def instance_preparation(hostname, depsdict, options) : ''' TBD ''' try : _status = 100 _store = False _process_manager = False _cleanup = False _fmsg = "An error has occurred, but no error message was captured" if "stores" in depsdict : for _store in depsdict["stores"] : _process_manager = ProcessManagement(hostname) _cmd = depsdict[_store + "_command_check"] _msg = "Checking accesss from this instance to the CB" _msg += " Orchestrator's " + _store.capitalize() + " with" _msg += " \"" + _cmd + "\"..." cbinfo(_msg) _status, _x, _y = _process_manager.run_os_command(_cmd) _msg = "This instance was able to access CB Orchestrator's " + _store.capitalize() _msg += " with \"" + _cmd + "\" (OK)" cbinfo(_msg) if _store.lower() == "filestore" : print '\n' _msg = "rsync rsync://" + depsdict["Filestore_ip"] + ':' + depsdict["Filestore_port"] + '/' + depsdict["Filestore_username"] + "_cb" print _msg print '\n' _msg = "--filestore " + depsdict["Filestore_ip"] + '-' + depsdict["Filestore_port"] + '-' + depsdict["Filestore_username"] print _msg print '\n' if str(options.addr) != "bypass" : _cmd = options.wksdir + "/common/cb_cleanup.sh" _msg = "Running the instance cleanup script \"" + _cmd + "\"..." cbinfo(_msg) _process_manager.run_os_command(_cmd) _cleanup = True _status = 0 except Exception, e : _status = 23 _fmsg = str(e)
def additional_host_discovery (self, obj_attr_list) : ''' TBD ''' _status = 100 _fmsg = "An error has occurred, but no error message was captured" try : if "additional_discovery" in obj_attr_list : if len(obj_attr_list["additional_discovery"]) > 1 : _proc_man = ProcessManagement(username = obj_attr_list["username"], \ cloud_name = obj_attr_list["cloud_name"]) _cmd = obj_attr_list["additional_discovery"] _cmd = _cmd.replace("--"," --") _status, _result_stdout, _result_stderr = _proc_man.run_os_command(_cmd) _extra_attr_list = json.loads(_result_stdout) for _host_uuid in obj_attr_list["hosts"].split(',') : if obj_attr_list["host_list"][_host_uuid]["cloud_hostname"] in _extra_attr_list : obj_attr_list["host_list"][_host_uuid].update(_extra_attr_list[obj_attr_list["host_list"][_host_uuid]["cloud_hostname"]]) _status = 0 else : _status = 0 else : _status = 0 except Exception, e : _status = 23 _fmsg = str(e)
def check_redis_binary(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking Redis version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "redis-server -v") if not _status and _result_stdout.count("Redis server"): _version = "N/A" for _word in _result_stdout.split(): if _word.count("v="): _version = _word.replace("v=", '') break elif _word.count('.') == 2: _version = _word break _msg += compare_versions('2.5.0', _version) _status = 0 else: _status = 1728289 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def check_redis_binary(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking Redis version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("redis-server -v") if not _status and _result_stdout.count("Redis server") : _version = "N/A" for _word in _result_stdout.split() : if _word.count("v=") : _version = _word.replace("v=",'') break elif _word.count('.') == 2 : _version = _word break _msg += compare_versions('2.5.0', _version) _status = 0 else : _status = 1728289 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def check_gmond_version(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking gmond version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("gmond --version") if not _status and _result_stdout.count("gmond") : for _word in _result_stdout.split() : if _word.count('.') >= 2 : _parts = _word.split(".") _version = _parts[0] + "." + _parts[1] break _msg += compare_versions('3.0', _version) _status = 0 else : _status = 1728289 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def check_rsyslogd_version(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking rsyslog version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("rsyslogd -v") if not _status and _result_stdout.count("compiled with") : _version = "N/A" for _word in _result_stdout.split() : if _word.count(".") and not _word.count("//") : _version = _word.replace(',','') break _msg += compare_versions('4.6.0', _version) _status = 0 else : _status = 1728289 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def check_mongo_binary(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking MongoDB version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("mongod --version") if not _status and _result_stdout.count("db version") : _version = "N/A" for _word in _result_stdout.split() : if _word.count("v") and not _word.count("version") : _version = _word.replace('v','').replace(',','') break _msg += compare_versions('2.0.0', _version) _status = 0 else : _status = 1728289 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def additional_host_discovery(self, obj_attr_list): ''' TBD ''' _status = 100 _fmsg = "An error has occurred, but no error message was captured" try: if "additional_discovery" in obj_attr_list: if len(obj_attr_list["additional_discovery"]) > 1: _proc_man = ProcessManagement(username = obj_attr_list["username"], \ cloud_name = obj_attr_list["cloud_name"]) _cmd = obj_attr_list["additional_discovery"] _cmd = _cmd.replace("--", " --") _status, _result_stdout, _result_stderr = _proc_man.run_os_command( _cmd) _extra_attr_list = json.loads(_result_stdout) for _host_uuid in obj_attr_list["hosts"].split(','): if obj_attr_list["host_list"][_host_uuid][ "cloud_hostname"] in _extra_attr_list: obj_attr_list["host_list"][_host_uuid].update( _extra_attr_list[obj_attr_list["host_list"][ _host_uuid]["cloud_hostname"]]) _status = 0 else: _status = 0 else: _status = 0 except Exception, e: _status = 23 _fmsg = str(e)
def check_mongo_binary(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking MongoDB version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "mongod --version") if not _status and _result_stdout.count("db version"): _version = "N/A" for _word in _result_stdout.split(): if _word.count("v") and not _word.count("version"): _version = _word.replace('v', '').replace(',', '') break _msg += compare_versions('2.0.0', _version) _status = 0 else: _status = 1728289 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def check_gmond_version(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking gmond version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "gmond --version") if not _status and _result_stdout.count("gmond"): for _word in _result_stdout.split(): if _word.count('.') >= 2: _parts = _word.split(".") _version = _parts[0] + "." + _parts[1] break _msg += compare_versions('3.0', _version) _status = 0 else: _status = 1728289 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def check_rsyslogd_version(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking rsyslog version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "rsyslogd -v") if not _status and _result_stdout.count("compiled with"): _version = "N/A" for _word in _result_stdout.split(): if _word.count(".") and not _word.count("//"): _version = _word.replace(',', '') break _msg += compare_versions('4.6.0', _version) _status = 0 else: _status = 1728289 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def check_rsync_version(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking rsync version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "rsync --version | grep version") if not _status and _result_stdout.count("protocol"): for _word in _result_stdout.split(): if _word.count('.') == 2: _version = _word break _msg += compare_versions('2.6', _version) _status = 0 else: _status = 1728289 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def vmccleanup(self, obj_attr_list) : ''' TBD ''' try : _status = 100 _fmsg = "An error has occurred, but no error message was captured" _curr_tries = 0 _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) sleep(_wait) self.common_messages("VMC", obj_attr_list, "cleaning up vms", 0, '') _running_instances = True while _running_instances and _curr_tries < _max_tries : _running_instances = False for _endpoint in self.lxdconn : _proc_man = ProcessManagement(username = "******", \ hostname = _endpoint, \ cloud_name = obj_attr_list["cloud_name"]) _cmd = "sudo pkill -9 -f 'rinetd -c /tmp/cb'; sudo rm -rf /tmp/cb-*.rinetd.conf" _status, _result_stdout, _fmsg = _proc_man.run_os_command(_cmd, raise_exception=False) _container_list = self.lxdconn[_endpoint].containers.all() for _container in _container_list : if _container.name.count("cb-" + obj_attr_list["username"] + '-' + obj_attr_list["cloud_name"]) : _running_instances = True _msg = "Terminating instance: " _msg += self.generate_random_uuid(_container.name) + " (" + str(_container.name) + ")" cbdebug(_msg, True) if _container.status == "Running" : _container.stop() _container.delete() sleep(_wait) _curr_tries += 1 if _curr_tries > _max_tries : _status = 1077 _fmsg = "Some instances on VMC \"" + obj_attr_list["name"] + "\"" _fmsg += " could not be removed because they never became active" _fmsg += ". They will have to be removed manually." cberr(_msg, True) else : _status = 0 except CldOpsException, obj : _status = obj.status _fmsg = str(obj.msg)
def vmdestroy(self, obj_attr_list): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _time_mark_drs = int(time()) if "mgt_901_deprovisioning_request_originated" not in obj_attr_list: obj_attr_list[ "mgt_901_deprovisioning_request_originated"] = _time_mark_drs obj_attr_list["mgt_902_deprovisioning_request_sent"] = \ _time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"]) self.common_messages("VM", obj_attr_list, "destroying", 0, '') if obj_attr_list["cloud_ip"] != "undefined" and obj_attr_list[ "role"] != "check": _proc_man = ProcessManagement(username = obj_attr_list["login"], \ cloud_name = obj_attr_list["cloud_name"], \ hostname = obj_attr_list["cloud_ip"], \ priv_key = obj_attr_list["identity"]) _cmd = "~/cb_cleanup.sh; rm ~/cb_*" _msg = "Shutting down CloudBench Load Manager/Metric Aggregator on " _msg += "VM \"" + obj_attr_list["name"] + "\" by executing the " _msg += "command \"" + _cmd + "\"" cbdebug(_msg, True) _status, _result_stdout, _result_stderr = _proc_man.run_os_command( _cmd) _time_mark_drc = int(time()) obj_attr_list["mgt_903_deprovisioning_request_completed"] = \ _time_mark_drc - _time_mark_drs _status = 0 except CldOpsException as obj: _status = obj.status _fmsg = str(obj.msg) except Exception as e: _status = 23 _fmsg = str(e) finally: _status, _msg = self.common_messages("VM", obj_attr_list, "destroyed", _status, _fmsg) return _status, _msg
def execute_command(operation, depkey, depsdict, hostname = "127.0.0.1", username = None, process_manager = None): ''' TBD ''' try : _status = 100 _msg = "Obtaining command to be executed...." _cmd = {} _cmd["configure"] = get_cmdline(depkey, depsdict, "configure") _cmd["install"] = get_cmdline(depkey, depsdict, "install") if not process_manager : process_manager = ProcessManagement(hostname) _order = depsdict[depkey + "-order"] if depkey != "sudo" and operation == "configure" : _msg = "(" + _order + ") Checking \"" + depkey + "\" version by executing the command \"" _msg += _cmd[operation] + "\"..." elif depkey == "sudo" and operation == "configure" : _msg = "(" + _order + ") Checking passwordless sudo for the user \"" + username + "\" " _msg += "by executing the command \"" + _cmd[operation] + "\"..." else : _msg = "(" + _order + ") Installing \"" + depkey + "\" by executing the command \"" _msg += _cmd[operation] + "\"..." print _msg _msg = "RESULT: " if depkey == "repo" and operation == "install" : build_repository_files(depsdict) _status, _result_stdout, _result_stderr = process_manager.run_os_command(_cmd[operation], False) if not _status : if operation == "install" : _msg += "DONE OK.\n" else : _msg += compare_versions(depkey, depsdict, _result_stdout.strip()) else : print _result_stderr _msg += "NOT OK. " if _msg.count("NOT OK") : _status = 701 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _result_stderr = str(obj.msg) _msg += "NOT OK. "
def configure_port_mapping(self, obj_attr_list, operation): ''' TBD ''' _status = 189 _fmsg = "About to configure port mapping" # LXD does not provide an automated method to expose specific ports # directly through the host's IP, like Docker does. For now, will # resort to ssh into the host and start a new "rinetd" instance each # time a new vmattach is issued. try: _proc_man = ProcessManagement(username = "******", \ hostname = obj_attr_list["host_cloud_ip"], \ cloud_name = obj_attr_list["cloud_name"]) if operation == "setup": _cmd = "echo \"0.0.0.0 " + obj_attr_list[ "prov_cloud_port"] + ' ' _cmd += obj_attr_list["cloud_ip"] + " 22\" > /tmp/" _cmd += obj_attr_list[ "cloud_vm_name"] + ".rinetd.conf; rinetd -c " _cmd += "/tmp/" + obj_attr_list[ "cloud_vm_name"] + ".rinetd.conf" _rexcpt = True else: _cmd = "sudo pkill -9 -f 'rinetd -c /tmp/" + obj_attr_list[ "cloud_vm_name"] _cmd += ".rinetd.conf" + "'; sudo rm -rf /tmp/" _cmd += obj_attr_list["cloud_vm_name"] + ".rinetd.conf" _rexcpt = False _msg = operation.capitalize( ) + " port mapping (" + obj_attr_list["prov_cloud_port"] _msg += " -> 22) for " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_vm_uuid"] + ") " _msg += "running on LXD host \"" + obj_attr_list["host_name"] + "\"" cbdebug(_msg, True) _status, _result_stdout, _fmsg = _proc_man.run_os_command( _cmd, raise_exception=_rexcpt) _status = 0 except ProcessManagement.ProcessManagementException, obj: _status = obj.status _fmsg = str(obj.msg)
def vmdestroy(self, obj_attr_list): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _time_mark_drs = int(time()) if "mgt_901_deprovisioning_request_originated" not in obj_attr_list: obj_attr_list[ "mgt_901_deprovisioning_request_originated"] = _time_mark_drs obj_attr_list["mgt_902_deprovisioning_request_sent"] = \ _time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"]) _msg = "Sending a termination request for " + obj_attr_list[ "name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_vm_uuid"] + ")" _msg += "...." cbdebug(_msg, True) _proc_man = ProcessManagement(username = obj_attr_list["login"], \ cloud_name = obj_attr_list["cloud_name"], \ hostname = obj_attr_list["cloud_ip"], \ priv_key = obj_attr_list["identity"]) _cmd = "~/cb_cleanup.sh; rm ~/cb_*" _msg = "Shutting down CloudBench Load Manager/Metric Aggregator on " _msg += "VM \"" + obj_attr_list["name"] + "\" by executing the " _msg += "command \"" + _cmd + "\"" cbdebug(_msg, True) _status, _result_stdout, _result_stderr = _proc_man.run_os_command( _cmd) _time_mark_drc = int(time()) obj_attr_list["mgt_903_deprovisioning_request_completed"] = \ _time_mark_drc - _time_mark_drs _status = 0 except CldOpsException, obj: _status = obj.status _fmsg = str(obj.msg)
def check_ifconfig_utility(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking \"ifconfig\" utility....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("ifconfig") if not _status : _msg += "OK" except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def check_ifconfig_utility(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking \"ifconfig\" utility....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "ifconfig") if not _status: _msg += "OK" except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def vmdestroy(self, obj_attr_list): """ TBD """ try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _time_mark_drs = int(time()) if "mgt_901_deprovisioning_request_originated" not in obj_attr_list: obj_attr_list["mgt_901_deprovisioning_request_originated"] = _time_mark_drs obj_attr_list["mgt_902_deprovisioning_request_sent"] = _time_mark_drs - int( obj_attr_list["mgt_901_deprovisioning_request_originated"] ) _msg = "Sending a termination request for " + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ")" _msg += "...." cbdebug(_msg, True) _proc_man = ProcessManagement( username=obj_attr_list["login"], cloud_name=obj_attr_list["cloud_name"], hostname=obj_attr_list["cloud_ip"], priv_key=obj_attr_list["identity"], ) _cmd = "~/cb_cleanup.sh; rm ~/cb_*" _msg = "Shutting down CloudBench Load Manager/Metric Aggregator on " _msg += 'VM "' + obj_attr_list["name"] + '" by executing the ' _msg += 'command "' + _cmd + '"' cbdebug(_msg, True) _status, _result_stdout, _result_stderr = _proc_man.run_os_command(_cmd) _time_mark_drc = int(time()) obj_attr_list["mgt_903_deprovisioning_request_completed"] = _time_mark_drc - _time_mark_drs _status = 0 except CldOpsException, obj: _status = obj.status _fmsg = str(obj.msg)
def vmdestroy(self, obj_attr_list) : ''' TBD ''' try : _status = 100 _fmsg = "An error has occurred, but no error message was captured" _time_mark_drs = int(time()) if "mgt_901_deprovisioning_request_originated" not in obj_attr_list : obj_attr_list["mgt_901_deprovisioning_request_originated"] = _time_mark_drs obj_attr_list["mgt_902_deprovisioning_request_sent"] = \ _time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"]) self.common_messages("VM", obj_attr_list, "destroying", 0, '') if obj_attr_list["cloud_ip"] != "undefined" and obj_attr_list["role"] != "check" : _proc_man = ProcessManagement(username = obj_attr_list["login"], \ cloud_name = obj_attr_list["cloud_name"], \ hostname = obj_attr_list["cloud_ip"], \ priv_key = obj_attr_list["identity"]) _cmd = "~/cb_cleanup.sh; rm ~/cb_*" _msg = "Shutting down CloudBench Load Manager/Metric Aggregator on " _msg += "VM \"" + obj_attr_list["name"] + "\" by executing the " _msg += "command \"" + _cmd + "\"" cbdebug(_msg, True) _status, _result_stdout, _result_stderr = _proc_man.run_os_command(_cmd) _time_mark_drc = int(time()) obj_attr_list["mgt_903_deprovisioning_request_completed"] = \ _time_mark_drc - _time_mark_drs _status = 0 except CldOpsException, obj : _status = obj.status _fmsg = str(obj.msg)
def configure_port_mapping(self, obj_attr_list, operation) : ''' TBD ''' _status = 189 _fmsg = "About to configure port mapping" # LXD does not provide an automated method to expose specific ports # directly through the host's IP, like Docker does. For now, will # resort to ssh into the host and start a new "rinetd" instance each # time a new vmattach is issued. try : _proc_man = ProcessManagement(username = "******", \ hostname = obj_attr_list["host_cloud_ip"], \ cloud_name = obj_attr_list["cloud_name"]) if operation == "setup" : _cmd = "echo \"0.0.0.0 " + obj_attr_list["prov_cloud_port"] + ' ' _cmd += obj_attr_list["cloud_ip"] + " 22\" > /tmp/" _cmd += obj_attr_list["cloud_vm_name"] + ".rinetd.conf; rinetd -c " _cmd += "/tmp/" + obj_attr_list["cloud_vm_name"] + ".rinetd.conf" _rexcpt = True else: _cmd = "sudo pkill -9 -f 'rinetd -c /tmp/" + obj_attr_list["cloud_vm_name"] _cmd += ".rinetd.conf" + "'; sudo rm -rf /tmp/" _cmd += obj_attr_list["cloud_vm_name"] + ".rinetd.conf" _rexcpt = False _msg = operation.capitalize() + " port mapping (" + obj_attr_list["prov_cloud_port"] _msg += " -> 22) for " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") " _msg += "running on LXD host \"" + obj_attr_list["host_name"] + "\"" cbdebug(_msg, True) _status, _result_stdout, _fmsg = _proc_man.run_os_command(_cmd, raise_exception = _rexcpt) _status = 0 except ProcessManagement.ProcessManagementException, obj: _status = obj.status _fmsg = str(obj.msg)
def check_passwordless_sudo(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking passwordless sudo for the user \"" + username + "\" ....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "sudo -S ls < /dev/null") if not _status: _msg += "Passwordless sudo checked OK" _status = 0 else: _status = 1728289 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def check_netcat(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking netcat (openbsd) version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("nc -v -w 1 localhost -z 22") if not _status : _version = "1.9" _msg += compare_versions('1.6', _version) _status = 0 else : _status = 1728289 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def check_git_version(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking git version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("git --version") if not _status and _result_stdout.count("git version") : _version = _result_stdout.replace("git version ",'').strip() _msg += compare_versions('1.6.0', _version) _status = 0 else : _status = 1728289 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def check_passwordless_sudo(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking passwordless sudo for the user \"" + username + "\" ....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("sudo -S ls < /dev/null") if not _status : _msg += "Passwordless sudo checked OK" _status = 0 else : _status = 1728289 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def check_netcat(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking netcat (openbsd) version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "nc -v -w 1 localhost -z 22") if not _status: _version = "1.9" _msg += compare_versions('1.6', _version) _status = 0 else: _status = 1728289 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def check_git_version(hostname, username, trd_party_dir): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking git version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command( "git --version") if not _status and _result_stdout.count("git version"): _version = _result_stdout.replace("git version ", '').strip() _msg += compare_versions('1.6.0', _version) _status = 0 else: _status = 1728289 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status)
def check_R_version(hostname, username, trd_party_dir) : ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _proc_man = ProcessManagement() _msg = "Checking R version....." _status, _result_stdout, _result_stderr = _proc_man.run_os_command("R --version | grep version | grep -v GNU") if not _status and _result_stdout.count("version") : for _word in _result_stdout.split() : if _word.count('.') == 2 : _version = _word break _msg += compare_versions('2.1', _version) _status = 0 else : _status = 1728289 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status)
def reset(global_objects, soft = True) : ''' TBD ''' try : _msg = "Killing all processes..." print _msg, _proc_man = ProcessManagement() _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f cbact") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f cloud-api") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f cloud-gui") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f ai-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f vm-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f submit-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f capture-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f gmetad.py") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f gtkCBUI_") print "done" _proc_man.run_os_command("screen -wipe") _msg = "Flushing Object Store..." print _msg, _rmc = RedisMgdConn(global_objects["objectstore"]) _rmc.flush_object_store() print "done" if not soft : _msg = "Flushing Metric Store..." print _msg, _mmc = MongodbMgdConn(global_objects["metricstore"]) _mmc.flush_metric_store(global_objects["mon_defaults"]["username"]) print "done" _msg = "Flushing Log Store..." print _msg, _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f rsyslogd") _file_list = [] _file_list.append("operations.log") _file_list.append("report.log") _file_list.append("submmiter.log") _file_list.append("loadmanager.log") _file_list.append("gui.log") _file_list.append("remotescripts.log") _file_list.append("monitor.log") _file_list.append("subscribe.log") for _fn in _file_list : _proc_man.run_os_command("rm -rf " + global_objects["space"]["stores_working_dir"] + "/logs/" + _fn) _status, _msg = syslog_logstore_setup(global_objects, "check") print "done" _msg = "" _status = 0 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg)
def vmdestroy(self, obj_attr_list) : ''' TBD ''' try : _status = 100 _fmsg = "An error has occurred, but no error message was captured" if "prov_cloud_ip" in obj_attr_list : try : _space_attr_list = self.osci.get_object(obj_attr_list["cloud_name"], "GLOBAL", False, "space", False) _data_dir = _space_attr_list["data_working_dir"] + "/" + obj_attr_list["experiment_id"] if not access(_data_dir, F_OK) : makedirs(_data_dir) _cmd = "scp -i " _cmd += _space_attr_list["ssh_key_name"] _cmd += " " + obj_attr_list["login"] + "@" + obj_attr_list["prov_cloud_ip"] _cmd += ":/home/" + obj_attr_list["login"] + "/nmon/nmon.csv " _cmd += _data_dir + "/nmon_" + obj_attr_list["cloud_vm_name"] + ".csv" print _cmd _proc_man = ProcessManagement() _proc_man.run_os_command(_cmd) except Exception, msg : cbdebug(str(msg), True) _time_mark_drs = int(time()) if "mgt_901_deprovisioning_request_originated" not in obj_attr_list : obj_attr_list["mgt_901_deprovisioning_request_originated"] = _time_mark_drs if not self.cskconn : self.connect(obj_attr_list["access"], obj_attr_list["api_key"], \ obj_attr_list["secret_key"]) _zoneid = self.get_zone(obj_attr_list["vmc_name"]) _wait = int(obj_attr_list["update_frequency"]) _instance = self.get_vm_instance(obj_attr_list) obj_attr_list["mgt_902_deprovisioning_request_sent"] = \ _time_mark_drs - int(obj_attr_list["mgt_901_deprovisioning_request_originated"]) if _instance : _msg = "Sending a termination request for " + obj_attr_list["name"] + "" _msg += " (instance id " + _instance[u'id'] + ")" _msg += "...." cbdebug(_msg, True) self.cskconn.destroyVirtualMachine({u'id': _instance[u'id']}) sleep(_wait) while self.is_vm_running(obj_attr_list) : sleep(_wait) _time_mark_drc = int(time()) obj_attr_list["mgt_903_deprovisioning_request_completed"] = \ _time_mark_drc - _time_mark_drs else : True # This needs to be changed later. I could not find an easy way to # find the actual volume id of a given instance _volumes = self.cskconn.listVolumes({u'zoneid': _zoneid}) if _volumes and len(_volumes) : for unattachedvol in _volumes : if unattachedvol[u'state'] == u'Allocated' : _msg = unattachedvol[u'id'] + ' ' + unattachedvol[u'state'] _msg += "... was deleted" cbdebug(_msg) self.cskconn.deleteVolume({u'id':unattachedvol[u'id']}) else: _msg = unattachedvol[u'id'] + ' ' + unattachedvol[u'state'] _msg += "... still attached and could not be deleted" cbdebug(_msg) else : _msg = "No volumes to remove" cbdebug(_msg) _status = 0
def dependency_checker_installer(hostname, depsdict, username, operation, options) : ''' TBD ''' try : _status = 100 _dep_missing = -1 _fmsg = "An error has occurred, but no error message was captured" if len(options.wks) > 1 : if options.wks.count("_ycsb") : options.wks += ",ycsb" if options.wks.count(",ycsb") : options.wks += ",mongo_ycsb,cassandra_ycsb,redis_ycsb" if options.wks.count(",acmeair") : options.wks += ",mongo_acmeair" deps_file_parser(depsdict, username, options, "127.0.0.1") docker_file_parser(depsdict, username, options, "127.0.0.1") preparation_file_parser(depsdict, username, options, "127.0.0.1") if "Filestore_ip" not in depsdict : depsdict["Filestore_ip"], depsdict["Filestore_port"], depsdict["Filestore_username"] = options.filestore.split('-') depsdict["cdistkind"], depsdict["cdistver"], depsdict["cdistmajorver"], depsdict["cdistnam"], depsdict["carch"] = get_linux_distro() depsdict["3rdpartydir"] = options.tpdir depsdict["scriptsdir"] = options.wksdir depsdict["credentialsdir"] = options.creddir depsdict["username"] = username if options.addr : depsdict["repo-addr1"] = options.addr depsdict["pip-addr1"] = options.addr if depsdict["carch"] == "x86_64" : depsdict["carch1"] = "x86_64" depsdict["carch2"] = "x86-64" depsdict["carch3"] = "amd64" elif depsdict["carch"] == "ppc64le" : depsdict["carch1"] = "ppc64le" depsdict["carch2"] = "ppc64" depsdict["carch3"] = "ppc64" else: depsdict["carch1"] = "aarch64" depsdict["carch2"] = "aarch64" depsdict["carch3"] = "aarch64" _missing_dep = [] _dep_list = [0] * 5000 if str(options.addr) != "bypass" : select_url("repo", depsdict) select_url("pip", depsdict) _raise_exception = True else : depsdict["pip_addr"] = None depsdict["repo_addr"] = None _raise_exception = False for _key in depsdict.keys() : if _key.count("-order") : _dependency = _key.replace("-order",'') _order = int(depsdict[_key]) * 20 _dep_list.insert(_order, _dependency) _dep_list = [x for x in _dep_list if x != 0] print '\n' if options.role.count("workload") : options.tag = "base," + options.role _msg = "##### This node will be used to play a role in the Virtual Applications" _msg += " (AIs) \"" + str(options.wks) + "\". Only a subset of the depedencies" _msg += " will be " + operation + "ed. This node cannot be used as an Orchestrator Node\n" _msg += "\n" cbinfo(_msg) else : options.tag = "base," + options.role + ',' + options.clouds _msg = "##### This node will be prepared as an Orchestration Node." _msg += " The full set of dependencies will be " + operation + "ed. " _msg += "\n" cbinfo(_msg) options.tag = options.tag.split(',') _selected_dep_list = [] for _dep in _dep_list : for _tag in options.tag : if _dep + "-tag" in depsdict : _dep_tag_list = depsdict[_dep + "-tag"].split(',') else : _dep_tag_list = [ "workload" ] if _tag in _dep_tag_list : if _dep not in _selected_dep_list : _selected_dep_list.append(_dep) _dep_list = _selected_dep_list _process_manager = ProcessManagement(hostname) _status, _std_out, _y = _process_manager.run_os_command("sudo cat /proc/1/cgroup | grep -c docker", raise_exception = False) if _status : depsdict["indocker"] = False else : if str(_std_out.replace("\n",'')) == '0' : depsdict["indocker"] = False else : depsdict["indocker"] = True _msg = "##### DETECTED OPERATING SYSTEM KIND: " + depsdict["cdistkind"] cbinfo(_msg) _msg = "##### DETECTED OPERATING SYSTEM VERSION: " + depsdict["cdistver"] + " (" + depsdict["cdistmajorver"] + ')' cbinfo(_msg) _msg = "##### DETECTED OPERATING SYSTEM NAME: " + depsdict["cdistnam"] cbinfo(_msg) _msg = "##### DETECTED ARCHITECTURE: " + depsdict["carch"] cbinfo(_msg) _msg = "##### DETECTED RUNNING INSIDE DOCKER: " + str(depsdict["indocker"]) cbinfo(_msg) print '\n' if operation == "configure" : if "repo" in _dep_list : _dep_list.remove("repo") if depsdict["cdistkind"] == "AMI" : _msg = "This node runs the \"" + depsdict["cdistkind"] + "\" Linux " _msg += "distribution. Will treat it as \"rhel\", but will disable" _msg += " the repository manipulation." cbinfo(_msg) depsdict["cdistkind"] = "rhel" if "repo" in _dep_list : _dep_list.remove("repo") if depsdict["carch"].count("ppc") and "mongdob" in _dep_list : _msg = "##### The processors on this node have a \"Power\" architecture." _msg += "Removing MongoDB and Chef (client) from the dependency list" cbwarn(_msg) _dep_list.remove("mongodb") _dep_list.remove("chef-client") if "java" in _dep_list and "oraclejava" in _dep_list : _msg = "Since both \"java\" and \"oraclejava\" are listed as dependencies" _msg += ", only \"oraclejava\" will be used" cbinfo(_msg) _dep_list.remove("java") _dep_list.remove("java-home") _fmsg = "" _dep_missing = 0 for _dep in _dep_list : _status, _msg = execute_command("configure", _dep, depsdict, \ hostname = "127.0.0.1", \ username = username, \ venv = options.venv, \ raise_exception = _raise_exception) if _status : _dep_missing += 1 _missing_dep.append(_dep) cberr(_msg) if operation == "install" : _status, _msg = execute_command("install", _dep, depsdict, \ hostname = "127.0.0.1", \ username = username, \ venv = options.venv, \ raise_exception = _raise_exception) if not _status : _dep_missing -= 1 _missing_dep.remove(_dep) cbinfo(_msg) else : cberr(_msg) else : cbinfo(_msg) _status = _dep_missing _fmsg += ','.join(_missing_dep) except KeyError, e: _status = 22 _fmsg = "Unable to find entry " + str(e) + " in dependencies dictionary. Check you dependencies configuration file(s)"
else: _msg = "File \"" + _file + "\" IGNORED...." print _msg if not len(depsdict): _msg = "Error: None of the files on the list \"" + str(_file_name_list) _msg += "\" contained configuration statements" print _msg exit(9) if _cleanup_repos: if not process_manager: process_manager = ProcessManagement(hostname) process_manager.run_os_command("sudo rm -rf /tmp/repoupdated", False) return True def get_linux_distro(): ''' TBD ''' _linux_distro_name, _linux_distro_ver, _x = platform.linux_distribution() if _linux_distro_name.count("Red Hat"): _distro = "rhel" elif _linux_distro_name.count("Scientific Linux"): _distro = "rhel" elif _linux_distro_name.count("CentOS"): _distro = "rhel"
def reset(global_objects, soft = True) : ''' TBD ''' try : _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] _username = global_objects["space"]["username"] _logstore_username = global_objects["logstore"]["username"] _filestore_username = global_objects["filestore"]["username"] _filestore_config_file_fn = _stores_wk_dir + '/' + _filestore_username + "_rsync.conf" _msg = "Killing all processes..." print _msg, _proc_man = ProcessManagement() _proc_man.run_os_command("pkill -9 -u " + _username + " -f cbact") _proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-api") _proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-gui") _proc_man.run_os_command("pkill -9 -u " + _username + " -f ai-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f vm-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f submit-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f capture-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f gmetad.py") _proc_man.run_os_command("pkill -9 -u " + _username + " -f gtkCBUI_") print "done" _proc_man.run_os_command("screen -wipe") _proc_man.run_os_command("rm -rf /tmp/restart_cb*" + global_objects["logstore"]["username"]) _proc_man.run_os_command("rm -rf /tmp/*-*-*-*-*_avg_acc") _msg = "Flushing Object Store..." print _msg, _rmc = RedisMgdConn(global_objects["objectstore"]) _rmc.flush_object_store() print "done" _msg = "Flushing Log Store..." print _msg, _proc_man.run_os_command("pkill -9 -u " + _logstore_username + " -f rsyslogd") _file_list = [] _file_list.append("operations.log") _file_list.append("report.log") _file_list.append("submmiter.log") _file_list.append("loadmanager.log") _file_list.append("gui.log") _file_list.append("remotescripts.log") _file_list.append("messages.log") _file_list.append("monitor.log") _file_list.append("subscribe.log") for _fn in _file_list : _proc_man.run_os_command("rm -rf " + _log_dir + '/' + _logstore_username + '_' + _fn) _proc_man.run_os_command("touch " + _log_dir + '/' + _logstore_username + '_' + _fn) _status, _msg = syslog_logstore_setup(global_objects, "check") global_objects["logstore"]["just_restarted"] = True print "done\n" #_msg = "Flushing File Store..." #print _msg, #_proc_man.run_os_command("sudo pkill -9 -u root -f \"rsync --daemon --config " + _filestore_config_file_fn +"\"", raise_exception = False) #_proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _filestore_username + "_rsyncd.pid") #_status, _msg = rsync_filestore_setup(global_objects, "check") #print "done" if not soft : _msg = "Flushing Metric Store..." print _msg, _mmc = MongodbMgdConn(global_objects["metricstore"]) _mmc.flush_metric_store(global_objects["mon_defaults"]["username"]) print "done" _msg = "" _status = 0 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg)
def take_action_if_requested(self, obj_type, obj_attr_list, current_step): ''' TBD ''' if "staging" not in obj_attr_list : return if not obj_attr_list["staging"].count(current_step) : return if obj_attr_list["staging"] + "_complete" in obj_attr_list : return try : _status = 100 _fmsg = "An error has occurred, but no error message was captured" if obj_attr_list["staging"] == "pause_" + current_step : if current_step == "provision_originated" : obj_attr_list["last_known_state"] = "waiting for signal" _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) # Always subscribe for the VM channel, no matter the object _sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", "staging", _max_tries * _wait) if obj_type == "VM" and obj_attr_list["ai"] != "none" and current_step.count("all_vms") : _target_uuid = obj_attr_list["ai"] _target_name = obj_attr_list["ai_name"] _cloud_vm_uuid = obj_attr_list["cloud_vm_uuid"] else : _target_uuid = obj_attr_list["uuid"] _target_name = obj_attr_list["name"] _cloud_vm_uuid = _target_uuid self.osci.publish_message(obj_attr_list["cloud_name"], \ obj_type, \ "staging", \ _target_uuid + ";vmready;" + dic2str(obj_attr_list),\ 1, \ 3600) _msg = obj_type + ' ' + _cloud_vm_uuid + " (" _msg += _target_name + ") pausing on attach for continue signal ...." cbdebug(_msg, True) for _message in _sub_channel.listen() : _args = str(_message["data"]).split(";") if len(_args) != 3 : # cbdebug("Message is not for me: " + str(_args)) continue _id, _status, _info = _args if (_id == _target_uuid or _id == _target_name) and _status == "continue" : obj_attr_list[obj_attr_list["staging"] + "_complete"] = int(time()) if _info.count(":") : _add_obj_attr_list = str2dic(_info) obj_attr_list.update(_add_obj_attr_list) _status = 0 break _sub_channel.unsubscribe() _status = 0 elif obj_attr_list["staging"] == "execute_" + current_step : if current_step == "provision_originated" : obj_attr_list["last_known_state"] = "about to execute script" _proc_man = ProcessManagement(username = obj_attr_list["username"], \ cloud_name = obj_attr_list["cloud_name"]) _json_contents = copy.deepcopy(obj_attr_list) if obj_type == "AI" : _json_contents["vms"] = {} if "vms" in obj_attr_list and current_step != "deprovision_finished" : _vm_id_list = obj_attr_list["vms"].split(',') for _vm_id in _vm_id_list : _vm_uuid = _vm_id.split('|')[0] _vm_attr_list = self.osci.get_object(obj_attr_list["cloud_name"], "VM", False, _vm_uuid, False) _json_contents["vms"][_vm_attr_list["uuid"]] = _vm_attr_list obj_attr_list["execute_json_filename"] = "/tmp/" obj_attr_list["execute_json_filename"] += obj_attr_list["execute_json_filename_prefix"] obj_attr_list["execute_json_filename"] += "_vapp_" + obj_attr_list["cloud_name"] obj_attr_list["execute_json_filename"] += "_" + obj_attr_list["name"] + "_" obj_attr_list["execute_json_filename"] += obj_attr_list["uuid"] + ".json" _json_fh = open(obj_attr_list["execute_json_filename"], 'w') _json_fh.write(json.dumps(_json_contents, sort_keys = True, indent = 4)) _json_fh.close() _msg = "JSON contents written to " _msg += obj_attr_list["execute_json_filename"] + '.' cbdebug(_msg, True) _cmd = obj_attr_list["execute_script_name"] + ' ' _cmd += obj_attr_list["execute_json_filename"] _status, _result_stdout, _result_stderr = _proc_man.run_os_command(_cmd) _msg = "Command \"" + _cmd + "\" executed, with return code " + str(_status) cbdebug(_msg, True) obj_attr_list[obj_attr_list["staging"] + "_stdout"] = _result_stdout obj_attr_list[obj_attr_list["staging"] + "_stderr"] = _result_stderr obj_attr_list[obj_attr_list["staging"] + "_complete"] = int(time()) except self.osci.ObjectStoreMgdConnException, obj : _status = obj.status _fmsg = str(obj.msg)
def syslog_logstore_setup(global_objects, operation="check"): ''' TBD ''' _hostname = global_objects["logstore"]["hostname"] _protocol = global_objects["logstore"]["protocol"] _username = global_objects["logstore"]["username"] _usage = global_objects["logstore"]["usage"].lower() _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] try: _name, _ip = hostname2ip(_hostname) if operation == "check": if _usage == "shared": _hostport = int(global_objects["logstore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol): _proc_man = ProcessManagement(username="******") _rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd") _cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid " if not _rsyslog_pid: _msg = "Unable to detect a shared rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _usage = "private" _proc_man = ProcessManagement(username=_username) _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf" _cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid" if not access(_config_file_fn, F_OK): # File was deleted, but the rsyslog process is still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) if not access(_log_dir, W_OK): # The directory does not even exist, kill any rsyslog processes still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) _proc_man.run_os_command("sudo mkdir -p " + _log_dir + " && sudo chmod 777 " + _log_dir) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid: global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\ protocol = "udp") _hostport = int(global_objects["logstore"]["port"]) _config_file_contents = global_objects["logstore"][ "config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace( "DOLLAR", '$') _config_file_contents = _config_file_contents.replace( "RSYSLOG", "RSYSLOG_") _config_file_contents = _config_file_contents.replace( "REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace( "REPLLOGDIR", _log_dir) _config_file_contents = _config_file_contents.replace( "REPLUSER", _username + '_') _config_file_contents = _config_file_contents.replace( ';', '\n') _config_file_contents = _config_file_contents.replace( "--", ';') _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon(_cmd) if not _rsyslog_pid: _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents: if _line.count("UDPServerRun"): global_objects["logstore"]["port"] = _line.split( )[1] _hostport = int(global_objects["logstore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except socket.herror: _status = 1200 _msg = "The IP address \"" + _hostname + "\" - used by the rsyslog " _msg += " daemon - is not mapped to a Hostname. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except socket.gaierror: _status = 1200 _msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog" _msg += " daemon - is not mapped to an IP. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def deps_file_parser(depsdict, username, options, hostname, process_manager = False) : ''' TBD ''' _file_name_list = [] _file_name_list.append(options.defdir + "/PUBLIC_dependencies.txt") _cleanup_repos = False if len(options.wks) > 1 : _workloads_list = options.wks.split(',') for _workload in _workloads_list : _file_name_list.append(options.wksdir + '/' + _workload + "/dependencies.txt") _cleanup_repos = True _file_name_list.append(options.defdir + "/IBM_dependencies.txt") _file_name_list.append(options.defdir + "/SPEC_dependencies.txt") if len(options.custom) : _file_name_list.append(options.cusdir + '/' + options.custom) print('\n') for _file in _file_name_list : if os.access(_file, os.F_OK) : try: _fd = open(_file, 'r') _fc = _fd.readlines() _fd.close() _msg = "##### File \"" + _file + "\" opened and loaded...." cbinfo(_msg) for _line in _fc : _line = _line.strip() if _line.count("#",0,2) : _sstr = None elif len(_line) < 3 : _sstr = None elif _line.count(" = ") : _sstr = " = " elif _line.count(" =") : _sstr = " =" elif _line.count("= ") : _sstr = "= " elif _line.count("=") : _sstr = "=" else : _sstr = None if _sstr : _key, _value = _line.split(_sstr) _key = _key.strip() depsdict[_key] = _value except Exception as e : _msg = "##### Error reading file \"" + _file + "\":" + str(e) cberr(_msg) exit(4) else : _msg = "##### File \"" + _file + "\" IGNORED...." cbwarn(_msg) if not len(depsdict) : _msg = "##### None of the files on the list \"" + str(_file_name_list) _msg += "\" contained configuration statements" cberr(_msg) exit(9) if _cleanup_repos : if not process_manager : process_manager = ProcessManagement(hostname) process_manager.run_os_command("sudo rm -rf /tmp/repoupdated", False) return True
def take_action_if_requested(self, obj_type, obj_attr_list, current_step): ''' TBD ''' if "staging" not in obj_attr_list: return if not obj_attr_list["staging"].count(current_step): return if obj_attr_list["staging"] + "_complete" in obj_attr_list: return try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" if obj_attr_list["staging"] == "pause_" + current_step: _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) # Always subscribe for the VM channel, no matter the object _sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", "staging", _max_tries * _wait) if obj_type == "VM" and obj_attr_list[ "ai"] != "none" and current_step.count("all_vms"): _target_uuid = obj_attr_list["ai"] _target_name = obj_attr_list["ai_name"] _cloud_vm_uuid = obj_attr_list["cloud_vm_uuid"] else: _target_uuid = obj_attr_list["uuid"] _target_name = obj_attr_list["name"] _cloud_vm_uuid = _target_uuid self.osci.publish_message(obj_attr_list["cloud_name"], \ obj_type, \ "staging", \ _target_uuid + ";vmready;" + dic2str(obj_attr_list),\ 1, \ 3600) _msg = obj_type + ' ' + _cloud_vm_uuid + " (" _msg += _target_name + ") pausing on attach for continue signal ...." cbdebug(_msg, True) for _message in _sub_channel.listen(): _args = str(_message["data"]).split(";") if len(_args) != 3: # cbdebug("Message is not for me: " + str(_args)) continue _id, _status, _info = _args if (_id == _target_uuid or _id == _target_name) and _status == "continue": obj_attr_list[obj_attr_list["staging"] + "_complete"] = int(time()) if _info.count(":"): _add_obj_attr_list = str2dic(_info) obj_attr_list.update(_add_obj_attr_list) _status = 0 break _sub_channel.unsubscribe() _status = 0 elif obj_attr_list["staging"] == "execute_" + current_step: _proc_man = ProcessManagement(username = obj_attr_list["username"], \ cloud_name = obj_attr_list["cloud_name"]) _json_contents = copy.deepcopy(obj_attr_list) if obj_type == "AI": _json_contents["vms"] = {} _vm_id_list = obj_attr_list["vms"].split(',') for _vm_id in _vm_id_list: _vm_uuid = _vm_id.split('|')[0] _vm_attr_list = self.osci.get_object( obj_attr_list["cloud_name"], "VM", False, _vm_uuid, False) _json_contents["vms"][ _vm_attr_list["uuid"]] = _vm_attr_list obj_attr_list["execute_json_filename"] = "/tmp/" obj_attr_list["execute_json_filename"] += obj_attr_list[ "execute_json_filename_prefix"] obj_attr_list[ "execute_json_filename"] += "_vapp_" + obj_attr_list[ "cloud_name"] obj_attr_list["execute_json_filename"] += "_" + obj_attr_list[ "name"] + "_" obj_attr_list[ "execute_json_filename"] += obj_attr_list["uuid"] + ".json" _json_fh = open(obj_attr_list["execute_json_filename"], 'w') _json_fh.write( json.dumps(_json_contents, sort_keys=True, indent=4)) _json_fh.close() _msg = "JSON contents written to " _msg += obj_attr_list["execute_json_filename"] + '.' cbdebug(_msg, True) _cmd = obj_attr_list["execute_script_name"] + ' ' _cmd += obj_attr_list["execute_json_filename"] _status, _result_stdout, _result_stderr = _proc_man.run_os_command( _cmd) _msg = "Command \"" + _cmd + "\" executed, with return code " + str( _status) cbdebug(_msg, True) obj_attr_list[obj_attr_list["staging"] + "_complete"] = int(time()) except self.osci.ObjectStoreMgdConnException, obj: _status = obj.status _fmsg = str(obj.msg)
def vmccleanup(self, obj_attr_list): ''' TBD ''' try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" _curr_tries = 0 _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) sleep(_wait) self.common_messages("VMC", obj_attr_list, "cleaning up vms", 0, '') _running_instances = True while _running_instances and _curr_tries < _max_tries: _running_instances = False for _endpoint in self.lxdconn: _proc_man = ProcessManagement(username = "******", \ hostname = _endpoint, \ cloud_name = obj_attr_list["cloud_name"]) _cmd = "sudo pkill -9 -f 'rinetd -c /tmp/cb'; sudo rm -rf /tmp/cb-*.rinetd.conf" _status, _result_stdout, _fmsg = _proc_man.run_os_command( _cmd, raise_exception=False) _container_list = self.lxdconn[_endpoint].containers.all() for _container in _container_list: if _container.name.count("cb-" + obj_attr_list["username"] + '-' + obj_attr_list["cloud_name"]): _running_instances = True _msg = "Terminating instance: " _msg += self.generate_random_uuid( _container.name) + " (" + str( _container.name) + ")" cbdebug(_msg, True) if _container.status == "Running": _container.stop() _container.delete() sleep(_wait) _curr_tries += 1 if _curr_tries > _max_tries: _status = 1077 _fmsg = "Some instances on VMC \"" + obj_attr_list[ "name"] + "\"" _fmsg += " could not be removed because they never became active" _fmsg += ". They will have to be removed manually." cberr(_msg, True) else: _status = 0 except CldOpsException as obj: _status = obj.status _fmsg = str(obj.msg) except LXDError.LXDAPIException as obj: _status = 18127 _fmsg = str(obj) except Exception as e: _status = 23 _fmsg = str(e) finally: _status, _msg = self.common_messages("VMC", obj_attr_list, "cleaned up", _status, _fmsg) return _status, _msg
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc): ''' TBD ''' _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) _network_reachable = False _curr_tries = 0 if not _network_reachable: _msg = "Trying to establish network connectivity to " _msg += obj_attr_list["name"] + " (cloud-assigned uuid " _msg += obj_attr_list["cloud_vm_uuid"] + "), on IP address " _msg += obj_attr_list["prov_cloud_ip"] if str(obj_attr_list["use_jumphost"]).lower() == "false": _msg += "..." else: _msg += " via jumphost " + obj_attr_list["jumphost_ip"] + "..." obj_attr_list["check_boot_complete"] = "run_command_/bin/true" cbdebug(_msg, True) self.pending_set(obj_attr_list, _msg) sleep(_wait) while not _network_reachable and _curr_tries < _max_tries: _start_pooling = int(time()) if "async" not in obj_attr_list or str( obj_attr_list["async"]).lower() == "false": if threading.current_thread().abort: _msg = "VM Create Aborting..." _status = 123 raise CldOpsException(_msg, _status) if obj_attr_list["check_boot_complete"].count("tcp_on_"): _nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"]) _port_to_check = obj_attr_list[ "check_boot_complete"].replace("tcp_on_", '') _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish a TCP connection to port " _msg += str(_port_to_check) + " on address " _msg += obj_attr_list["prov_cloud_ip"] cbdebug(_msg) _vm_is_booted = _nh_conn.check_port( int(_port_to_check), "TCP") elif obj_attr_list["check_boot_complete"].count("cloud_ping"): _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish network connectivity " _msg += "through the cloud's API" cbdebug(_msg) _vm_is_booted = self.is_vm_alive(obj_attr_list) elif obj_attr_list["check_boot_complete"].count( "subscribe_on_"): _string_to_search = obj_attr_list["prov_cloud_ip"] + " is " _string_to_search += "booted" _channel_to_subscribe = obj_attr_list[ "check_boot_complete"].replace("subscribe_on_", '') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list[ "cloud_vm_uuid"] + ") has booted by " _msg += "subscribing to channel \"" + str( _channel_to_subscribe) _msg += "\" and waiting for the message \"" _msg += _string_to_search + "\"." cbdebug(_msg) self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) _sub_channel = self.osci.subscribe( obj_attr_list["cloud_name"], "VM", _channel_to_subscribe, _max_tries * _wait) for _message in _sub_channel.listen(): if str(_message["data"]).count(_string_to_search): _vm_is_booted = True break _sub_channel.unsubscribe() self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) elif obj_attr_list["check_boot_complete"].count("wait_for_"): _boot_wait_time = int( obj_attr_list["check_boot_complete"].replace( "wait_for_", '')) _msg = "Assuming that the VM \"" + obj_attr_list[ "cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted after" _msg += " waiting for " + str( _boot_wait_time) + " seconds." cbdebug(_msg) if _boot_wait_time: sleep(_boot_wait_time) _vm_is_booted = True elif obj_attr_list["check_boot_complete"].count( "run_command_"): _command_to_run = obj_attr_list[ "check_boot_complete"].replace("run_command_", '') _command_to_run = _command_to_run.replace("____", ' ') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list[ "cloud_vm_uuid"] + ") has booted by " _msg += "running the command \"" + str(_command_to_run) cbdebug(_msg) if _curr_tries <= _max_tries / 3: _connection_timeout = int( obj_attr_list["update_frequency"]) / 2 elif _curr_tries > _max_tries / 3 and _curr_tries < 2 * _max_tries / 3: _connection_timeout = int( obj_attr_list["update_frequency"]) obj_attr_list[ "comments"] += "Had to increase ssh timeout. " else: _connection_timeout = int( obj_attr_list["update_frequency"]) * 2 obj_attr_list[ "comments"] += "Had to increase ssh timeout one more time. " if str(obj_attr_list["use_jumphost"]).lower() == "true": if "ssh_config_file" in obj_attr_list: _ssh_conf_file = obj_attr_list["ssh_config_file"] else: _ssh_conf_file = None else: _ssh_conf_file = None _proc_man = ProcessManagement(username = obj_attr_list["login"], \ cloud_name = obj_attr_list["cloud_name"], \ hostname = obj_attr_list["prov_cloud_ip"], \ priv_key = obj_attr_list["identity"], \ config_file = _ssh_conf_file, connection_timeout = _connection_timeout) try: _status, _result_stdout, _result_stderr = _proc_man.run_os_command( _command_to_run) if not _status: _vm_is_booted = True else: _vm_is_booted = False except: _vm_is_booted = False elif obj_attr_list["check_boot_complete"].count( "snmpget_poll"): import netsnmp # Send SNMP GET message. Flag VM as booted if any response at all is recieved _vm_is_booted = False try: _msg = "Opening SNMP session to " + obj_attr_list[ "cloud_ip"] cbdebug(_msg) _snmp_wait_time = _wait * 1000000 _snmp_version = int(obj_attr_list["snmp_version"]) _snmp_comm = str(obj_attr_list["snmp_community"]) _snmp_session = netsnmp.Session(Version=_snmp_version, \ DestHost=obj_attr_list["cloud_ip"], \ Community=_snmp_comm, \ Timeout=_snmp_wait_time, Retries=0) _vars = netsnmp.VarList( netsnmp.Varbind(obj_attr_list["snmp_variable"], '0')) _snmp_response = _snmp_session.get(_vars) except: if _snmp_session.ErrorStr: _msg = "Error in SNMP handler : " + _snmp_session.ErrorStr else: _msg = "Unknown error in SNMP handler." cbdebug(_msg) _status = 200 raise CldOpsException(_msg, _status) if (_snmp_response[0] != None): _vm_is_booted = True _msg = "SNMP Response: " + str(_snmp_response) cbdebug(_msg) else: _vm_is_booted = False _msg = "Warning: No valid method specified to determined if VM has booted." cbdebug(_msg, True) _pooling_time = int(time()) - _start_pooling if _pooling_time <= _wait: _actual_wait = _wait - _pooling_time else: _msg = "The time spent on pooling for \"booted\" status (" + str( _pooling_time) _msg += " s) is actually longer than the " _msg += "interval between pooling attempts (" + str( _wait) + " s)." cbdebug(_msg, True) _actual_wait = 0 if _vm_is_booted: obj_attr_list["mgt_004_network_acessible"] = int( time()) - time_mark_prc self.pending_set(obj_attr_list, "Network accessible now. Continuing...") _network_reachable = True break else: _msg = "(" + str( _curr_tries) + ") " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_vm_uuid"] + ") " _msg += "still not network reachable. Will wait for " + str( _actual_wait) _msg += " seconds and check again." self.pending_set(obj_attr_list, _msg) cbdebug(_msg) sleep(_actual_wait) _curr_tries += 1 if _curr_tries < _max_tries: _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_vm_uuid"] + ") " _msg += "is network reachable (boot process finished successfully)" cbdebug(_msg) obj_attr_list["arrival"] = int(time()) # It should be mgt_006 and mgt_007 NOT mgt_005 obj_attr_list["mgt_006_instance_preparation"] = "0" obj_attr_list["mgt_007_application_start"] = "0" self.pending_set(obj_attr_list, "Application starting up...") self.get_attr_from_pending(obj_attr_list, "all") else: _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list[ "cloud_vm_uuid"] + ") " _msg += "is not network reachable after " + str( _max_tries * _wait) + " seconds.... " _msg += "Giving up." cberr(_msg, True) raise CldOpsException(_msg, 89)
def reset(global_objects, soft=True, cloud_name=None): ''' TBD ''' try: _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] _username = global_objects["space"]["username"] _logstore_username = global_objects["logstore"]["username"] _filestore_username = global_objects["filestore"]["username"] _filestore_config_file_fn = _stores_wk_dir + '/' + _filestore_username + "_rsync.conf" _msg = " Killing all processes..." print(_msg, end=' ') _proc_man = ProcessManagement() _proc_man.run_os_command("pkill -9 -u " + _username + " -f cbact") _proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-api") _proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-gui") _proc_man.run_os_command("pkill -9 -u " + _username + " -f ai-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f vm-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f submit-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f capture-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f gmetad.py") _proc_man.run_os_command("pkill -9 -u " + _username + " -f gtkCBUI_") print("done") _proc_man.run_os_command("screen -wipe") _proc_man.run_os_command( "rm -rf " + global_objects["space"]["generated_configurations_dir"] + '/' + _username + "_cb_lastcloudrc") _proc_man.run_os_command("rm -rf /tmp/restart_cb*" + global_objects["logstore"]["username"] + '*') _proc_man.run_os_command("rm -rf /tmp/" + _username + "_*-*-*-*-*_avg_acc") _msg = " Flushing Object Store..." print(_msg, end=' ') _rmc = RedisMgdConn(global_objects["objectstore"]) ''' If and only if the object store is being shared and a default 'STARTUP_CLOUD' or cloud on the command line has been provided, then we can selectively flush the database instead of wholly dropping it. ''' if global_objects["objectstore"]["usage"].lower() == "private": # Backwards-compatible behavior cloud_name = None _rmc.flush_object_store(cloud_name) print("done") _msg = " Flushing Log Store..." print(_msg, end=' ') if global_objects["logstore"]["usage"].lower() != "shared": _proc_man.run_os_command("pkill -9 -u " + _logstore_username + " -f rsyslogd") _file_list = [] _file_list.append("operations.log") _file_list.append("report.log") _file_list.append("submmiter.log") _file_list.append("loadmanager.log") _file_list.append("gui.log") _file_list.append("remotescripts.log") _file_list.append("messages.log") _file_list.append("monitor.log") _file_list.append("subscribe.log") for _fn in _file_list: if global_objects["logstore"]["usage"].lower() != "shared": _proc_man.run_os_command("rm -rf " + _log_dir + '/' + _logstore_username + '_' + _fn) _proc_man.run_os_command("touch " + _log_dir + '/' + _logstore_username + '_' + _fn) _status, _msg = syslog_logstore_setup(global_objects, "check") global_objects["logstore"]["just_restarted"] = True print("done") #_msg = "Flushing File Store..." #print _msg, #_proc_man.run_os_command("sudo pkill -9 -u root -f \"rsync --daemon --config " + _filestore_config_file_fn +"\"", raise_exception = False) #_proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _filestore_username + "_rsyncd.pid") #_status, _msg = rsync_filestore_setup(global_objects, "check") #print "done" if not soft: _msg = " Flushing Metric Store..." print(_msg, end=' ') _mmc = load_metricstore_adapter(global_objects["metricstore"]) _mmc.flush_metric_store(global_objects["mon_defaults"]["username"]) print("done") print('\n') _msg = "" _status = 0 except ProcessManagement.ProcessManagementException as obj: _status = str(obj.status) _msg = str(obj.msg) except MongodbMgdConn.MetricStoreMgdConnException as obj: _status = str(obj.status) _msg = str(obj.msg) except MysqlMgdConn.MetricStoreMgdConnException as obj: _status = str(obj.status) _msg = str(obj.msg) except RedisMgdConn.ObjectStoreMgdConnException as obj: _status = str(obj.status) _msg = str(obj.msg) except Exception as e: _status = 23 _msg = str(e) finally: return _status, _msg
def dependency_checker_installer(hostname, depsdict, username, operation, options): ''' TBD ''' try: _status = 100 _dep_missing = -1 _fmsg = "An error has occurred, but no error message was captured" if len(options.wks) > 1: if options.wks.count("_ycsb"): options.wks += ",ycsb" if options.wks.count(",ycsb"): options.wks += ",mongo_ycsb,cassandra_ycsb,redis_ycsb" if options.wks.count(",acmeair"): options.wks += ",mongo_acmeair" deps_file_parser(depsdict, username, options, "127.0.0.1") docker_file_parser(depsdict, username, options, "127.0.0.1") preparation_file_parser(depsdict, username, options, "127.0.0.1") if "Filestore_ip" not in depsdict: depsdict["Filestore_ip"], depsdict["Filestore_port"], depsdict[ "Filestore_username"] = options.filestore.split('-') depsdict["cdistkind"], depsdict["cdistver"], depsdict[ "cdistmajorver"], depsdict["cdistnam"], depsdict[ "carch"] = get_linux_distro() depsdict["3rdpartydir"] = options.tpdir depsdict["scriptsdir"] = options.wksdir depsdict["credentialsdir"] = options.creddir depsdict["username"] = username if options.addr: depsdict["repo-addr1"] = options.addr depsdict["pip-addr1"] = options.addr if depsdict["carch"] == "x86_64": depsdict["carch1"] = "x86_64" depsdict["carch2"] = "x86-64" depsdict["carch3"] = "amd64" elif depsdict["carch"] == "ppc64le": depsdict["carch1"] = "ppc64le" depsdict["carch2"] = "ppc64" depsdict["carch3"] = "ppc64" else: depsdict["carch1"] = "aarch64" depsdict["carch2"] = "aarch64" depsdict["carch3"] = "aarch64" _missing_dep = [] _dep_list = [0] * 5000 if str(options.addr) != "bypass": select_url("repo", depsdict) select_url("pip", depsdict) _raise_exception = True else: depsdict["pip_addr"] = None depsdict["repo_addr"] = None _raise_exception = False for _key in depsdict.keys(): if _key.count("-order"): _dependency = _key.replace("-order", '') _order = int(depsdict[_key]) * 20 _dep_list.insert(_order, _dependency) _dep_list = [x for x in _dep_list if x != 0] print '\n' if options.role.count("workload"): options.tag = "base," + options.role _msg = "##### This node will be used to play a role in the Virtual Applications" _msg += " (AIs) \"" + str( options.wks) + "\". Only a subset of the depedencies" _msg += " will be " + operation + "ed. This node cannot be used as an Orchestrator Node\n" _msg += "\n" cbinfo(_msg) else: options.tag = "base," + options.role + ',' + options.clouds _msg = "##### This node will be prepared as an Orchestration Node." _msg += " The full set of dependencies will be " + operation + "ed. " _msg += "\n" cbinfo(_msg) options.tag = options.tag.split(',') _selected_dep_list = [] for _dep in _dep_list: for _tag in options.tag: if _dep + "-tag" in depsdict: _dep_tag_list = depsdict[_dep + "-tag"].split(',') else: _dep_tag_list = ["workload"] if _tag in _dep_tag_list: if _dep not in _selected_dep_list: _selected_dep_list.append(_dep) _dep_list = _selected_dep_list _process_manager = ProcessManagement(hostname) _status, _std_out, _y = _process_manager.run_os_command( "sudo cat /proc/1/cgroup | grep -c docker", raise_exception=False) if _status: depsdict["indocker"] = False else: if str(_std_out.replace("\n", '')) == '0': depsdict["indocker"] = False else: depsdict["indocker"] = True _msg = "##### DETECTED OPERATING SYSTEM KIND: " + depsdict["cdistkind"] cbinfo(_msg) _msg = "##### DETECTED OPERATING SYSTEM VERSION: " + depsdict[ "cdistver"] + " (" + depsdict["cdistmajorver"] + ')' cbinfo(_msg) _msg = "##### DETECTED OPERATING SYSTEM NAME: " + depsdict["cdistnam"] cbinfo(_msg) _msg = "##### DETECTED ARCHITECTURE: " + depsdict["carch"] cbinfo(_msg) _msg = "##### DETECTED RUNNING INSIDE DOCKER: " + str( depsdict["indocker"]) cbinfo(_msg) print '\n' if operation == "configure": if "repo" in _dep_list: _dep_list.remove("repo") if depsdict["cdistkind"] == "AMI": _msg = "This node runs the \"" + depsdict["cdistkind"] + "\" Linux " _msg += "distribution. Will treat it as \"rhel\", but will disable" _msg += " the repository manipulation." cbinfo(_msg) depsdict["cdistkind"] = "rhel" if "repo" in _dep_list: _dep_list.remove("repo") if depsdict["carch"].count("ppc") and "mongdob" in _dep_list: _msg = "##### The processors on this node have a \"Power\" architecture." _msg += "Removing MongoDB and Chef (client) from the dependency list" cbwarn(_msg) _dep_list.remove("mongodb") _dep_list.remove("chef-client") if "java" in _dep_list and "oraclejava" in _dep_list: _msg = "Since both \"java\" and \"oraclejava\" are listed as dependencies" _msg += ", only \"oraclejava\" will be used" cbinfo(_msg) _dep_list.remove("java") _dep_list.remove("java-home") _fmsg = "" _dep_missing = 0 for _dep in _dep_list: _status, _msg = execute_command("configure", _dep, depsdict, \ hostname = "127.0.0.1", \ username = username, \ venv = options.venv, \ raise_exception = _raise_exception) if _status: _dep_missing += 1 _missing_dep.append(_dep) cberr(_msg) if operation == "install": _status, _msg = execute_command("install", _dep, depsdict, \ hostname = "127.0.0.1", \ username = username, \ venv = options.venv, \ raise_exception = _raise_exception) if not _status: _dep_missing -= 1 _missing_dep.remove(_dep) cbinfo(_msg) else: cberr(_msg) else: cbinfo(_msg) _status = _dep_missing _fmsg += ','.join(_missing_dep) except KeyError, e: _status = 22 _fmsg = "Unable to find entry " + str( e ) + " in dependencies dictionary. Check you dependencies configuration file(s)"
def syslog_logstore_setup(global_objects, operation = "check") : ''' TBD ''' _hostname = global_objects["logstore"]["hostname"] _protocol = global_objects["logstore"]["protocol"] _username = global_objects["logstore"]["username"] _usage = global_objects["logstore"]["usage"].lower() _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] try : _name, _ip = hostname2ip(_hostname) if operation == "check" : if _usage == "shared" : _hostport = int(global_objects["logstore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol) : _proc_man = ProcessManagement(username = "******") _rsyslog_pid = _proc_man.get_pid_from_cmdline("rsyslogd") _cmd = "/sbin/rsyslogd -i /var/run/syslogd.pid " if not _rsyslog_pid : _msg = "Unable to detect a shared rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _usage = "private" _proc_man = ProcessManagement(username = _username) _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf" _cmd = "rsyslogd -f " + _config_file_fn + " " + "-i " + _stores_wk_dir + "/rsyslog.pid" if not access(_config_file_fn, F_OK) : # File was deleted, but the rsyslog process is still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) if not access(_log_dir, W_OK) : # The directory does not even exist, kill any rsyslog processes still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) _proc_man.run_os_command("sudo mkdir -p " + _log_dir + " && sudo chmod 777 " + _log_dir) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid : global_objects["logstore"]["port"] = _proc_man.get_free_port(global_objects["logstore"]["port"],\ protocol = "udp") _hostport = int(global_objects["logstore"]["port"]) _config_file_contents = global_objects["logstore"]["config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace("DOLLAR", '$') _config_file_contents = _config_file_contents.replace("RSYSLOG", "RSYSLOG_") _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace("REPLLOGDIR", _log_dir) _config_file_contents = _config_file_contents.replace("REPLUSER", _username + '_') _config_file_contents = _config_file_contents.replace(';','\n') _config_file_contents = _config_file_contents.replace("--", ';') _config_file_fn = _stores_wk_dir + '/' + _username + "_rsyslog.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon(_cmd) if not _rsyslog_pid : _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents : if _line.count("UDPServerRun") : global_objects["logstore"]["port"] = _line.split()[1] _hostport = int(global_objects["logstore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A Log Store of the kind \"rsyslog\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except socket.herror: _status = 1200 _msg = "The IP address \"" + _hostname + "\" - used by the rsyslog " _msg += " daemon - is not mapped to a Hostname. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except socket.gaierror: _status = 1200 _msg = "The Hostname \"" + _hostname + "\" - used by the rsyslog" _msg += " daemon - is not mapped to an IP. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def expand_command(cmdline, depsdict, process_manager = False) : ''' TBD ''' _actual_cmdline = '' for _command in cmdline.split(';') : if _command.count("package_install") : _packages = _command.replace("package_install",'') if depsdict["cdistkind"] == "ubuntu" : if _packages.count(".deb") : _command = "sudo dpkg -i PACKAGES; sudo apt-get -f install -y --force-yes --allow-unauthenticated " else : _command = "sudo apt-get -q -y --force-yes --allow-unauthenticated -o Dpkg::Options::=\"--force-confnew\" install PACKAGES" elif depsdict["cdistkind"] == "rhel" or depsdict["cdistkind"] == "fedora" : if _packages.count(".rpm") : _command = "sudo rpm -i PACKAGES" else : _command = "sudo yum -y install PACKAGES" _command = _command.replace("PACKAGES", _packages) if _command.count("sudo pip ") : if depsdict["indocker"] : _command = _command.replace("sudo pip", "export LC_ALL=C; sudo pip") if _command.count("IF DOCKER") : if depsdict["indocker"] : _command = "/bin/true" else : _command = _command.replace("IF DOCKER", '') if _command.count("service_restart_enable") or _command.count("service_stop_disable") : if not process_manager : process_manager = ProcessManagement() if _command.count("service_restart_enable") : _services = _command.replace("service_restart_enable",'') if _command.count("service_stop_disable") : if depsdict["indocker"] : _services = '' else : _services = _command.replace("service_stop_disable",'') _command = '' for _service in _services.split() : _status = 20000 _result_stdout = "NA" _result_stderr = "NA" _status, _result_stdout, _result_stderr = \ process_manager.run_os_command("sudo systemctl status " + _service, \ raise_exception = False) if not _status : if cmdline.count("service_restart_enable") : _service_cmd="sudo systemctl restart SERVICE; sudo systemctl enable SERVICE" if cmdline.count("service_stop_disable") : _service_cmd="sudo systemctl stop SERVICE; sudo systemctl disable SERVICE" else : if cmdline.count("service_restart_enable") : _service_cmd="sudo service SERVICE restart;" if depsdict["cdistkind"] == "rhel" or depsdict["cdistkind"] == "fedora" : _service_cmd += "sudo chkconfig SERVICE on;" elif depsdict["cdistkind"] == "ubuntu" : if os.access("/etc/init/" + _service + ".conf", os.F_OK) : _service_cmd += "sudo rm -rf /etc/init/SERVICE.override" else : _service_cmd += "sudo update-rc.d -f SERVICE defaults" if cmdline.count("service_stop_disable") : _service_cmd="sudo systemctl SERVICE stop; sudo chkconfig SERVICE off;" if depsdict["cdistkind"] == "rhel" or depsdict["cdistkind"] == "fedora" : _service_cmd = "sudo chkconfig SERVICE off;" elif depsdict["cdistkind"] == "ubuntu" : if os.access("/etc/init/" + _service + ".conf", os.F_OK) : _service_cmd = "sudo sh -c 'echo manual > /etc/init/SERVICE.override'" else : _service_cmd = "sudo update-rc.d -f SERVICE remove" _service_cmd = _service_cmd.replace("SERVICE", _service) + ';' _command += _service_cmd _actual_cmdline += _command + ';' return _actual_cmdline
def rsync_filestore_setup(global_objects, operation = "check") : ''' TBD ''' _hostname = global_objects["filestore"]["hostname"] _protocol = global_objects["filestore"]["protocol"] _username = global_objects["filestore"]["username"] _port = global_objects["filestore"]["port"] _usage = global_objects["filestore"]["usage"].lower() _base_dir = global_objects["space"]["base_dir"] _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] try : _name, _ip = hostname2ip(_hostname) if operation == "check" : if _usage == "shared" : _hostport = int(global_objects["filestore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol) : _proc_man = ProcessManagement(username = "******") _rsync_pid = _proc_man.get_pid_from_cmdline("rsync --daemon") _cmd = "rsync --daemon" if not _rsync_pid : _msg = "Unable to detect a shared rsync server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _usage = "private" _proc_man = ProcessManagement(username = _username) _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf" _cmd = "rsync --daemon --config " + _config_file_fn if not access(_config_file_fn, F_OK) : # File was deleted, but the rsync process is still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid : _proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _username + "_rsyncd.pid") global_objects["filestore"]["port"] = _proc_man.get_free_port(global_objects["filestore"]["port"], protocol = "tcp") _hostport = int(global_objects["filestore"]["port"]) _config_file_contents = global_objects["filestore"]["config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace("DOLLAR", '$') _config_file_contents = _config_file_contents.replace("REPLEQUAL", '=') _config_file_contents = _config_file_contents.replace("REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace("REPLLOGDIR", _log_dir) _config_file_contents = _config_file_contents.replace("REPLUSERU", _username + '_') _config_file_contents = _config_file_contents.replace("REPLUSER", _username) _config_file_contents = _config_file_contents.replace("REPLBASEDIR", _base_dir) _config_file_contents = _config_file_contents.replace("REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace(';','\n') _config_file_contents = _config_file_contents.replace("--", ';') _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd) if not _rsyslog_pid : _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else : _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents : if _line.count("port=") : global_objects["filestore"]["port"] = _line.split('=')[1] _hostport = int(global_objects["filestore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A File Store of the kind \"rsync\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except socket.herror: _status = 1200 _msg = "The IP address \"" + _hostname + "\" - used by the rsync " _msg += " daemon - is not mapped to a Hostname. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except socket.gaierror: _status = 1200 _msg = "The Hostname \"" + _hostname + "\" - used by the rsync" _msg += " daemon - is not mapped to an IP. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def execute_command(operation, depkey, depsdict, hostname = "127.0.0.1", username = None, process_manager = None, venv = False, raise_exception = False): ''' TBD ''' try : _status = 100 _msg = "Obtaining command to be executed...." _status = 20000 _result_stdout = "NA" _result_stderr = "NA" if not process_manager : process_manager = ProcessManagement(hostname) _cmd = {} _cmd["configure-keys"], _cmd["configure"] = get_cmdline(depkey, depsdict, "configure", process_manager, raise_exception) _cmd["install-keys"], _cmd["install"] = get_cmdline(depkey, depsdict, "install", process_manager, raise_exception) _order = depsdict[depkey + "-order"] if depkey != "sudo" and operation == "configure" : _msg = "(" + _order + ") Checking \"" + depkey + "\" version by executing the command \"" _msg += _cmd[operation] + "\" (" + _cmd[operation + "-keys"] + ")..." elif depkey == "sudo" and operation == "configure" : _msg = "(" + _order + ") Checking passwordless sudo for the user \"" + username + "\" " _msg += "by executing the command \"" + _cmd[operation] + "\" (" + _cmd[operation + "-keys"] + ")..." else : if venv : _cmd["install"] = _cmd["install"].replace("sudo pip", "pip") _msg = "(" + _order + ") Installing \"" + depkey + "\" by executing the command \"" _msg += _cmd[operation] + "\" (" + _cmd[operation + "-keys"] + ")..." cbinfo(_msg) _msg = "(" + _order + ") RESULT for command \"" + _cmd[operation] + "\" : " if depkey == "repo" and operation == "install" : build_repository_files(depsdict) _status, _result_stdout, _result_stderr = process_manager.run_os_command(_cmd[operation], False, True, False, True, None, False, 22, False) if not _status : if operation == "install" : _msg += "DONE OK.\n" else : _msg += compare_versions(depkey, depsdict, _result_stdout.strip()) else : _msg += "NOT OK (exit code " + str(_status) + "). " if _msg.count("NOT OK") : _status = 701 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _result_stderr = str(obj.msg) _msg += "NOT OK (PMgr Exception, exit code " + str(_status) + "). "
def execute_command(operation, depkey, depsdict, hostname="127.0.0.1", username=None, process_manager=None, venv=False, raise_exception=False): ''' TBD ''' try: _status = 100 _msg = "Obtaining command to be executed...." _status = 20000 _result_stdout = "NA" _result_stderr = "NA" if not process_manager: process_manager = ProcessManagement(hostname) _cmd = {} _cmd["configure-keys"], _cmd["configure"] = get_cmdline( depkey, depsdict, "configure", process_manager, raise_exception) _cmd["install-keys"], _cmd["install"] = get_cmdline( depkey, depsdict, "install", process_manager, raise_exception) _order = depsdict[depkey + "-order"] if depkey != "sudo" and operation == "configure": _msg = "(" + _order + ") Checking \"" + depkey + "\" version by executing the command \"" _msg += _cmd[operation] + "\" (" + _cmd[operation + "-keys"] + ")..." elif depkey == "sudo" and operation == "configure": _msg = "(" + _order + ") Checking passwordless sudo for the user \"" + username + "\" " _msg += "by executing the command \"" + _cmd[ operation] + "\" (" + _cmd[operation + "-keys"] + ")..." else: if venv: _cmd["install"] = _cmd["install"].replace("sudo pip", "pip") _msg = "(" + _order + ") Installing \"" + depkey + "\" by executing the command \"" _msg += _cmd[operation] + "\" (" + _cmd[operation + "-keys"] + ")..." cbinfo(_msg) _msg = "(" + _order + ") RESULT for command \"" + _cmd[ operation] + "\" : " if depkey == "repo" and operation == "install": build_repository_files(depsdict) _status, _result_stdout, _result_stderr = process_manager.run_os_command( _cmd[operation], False, True, False, True, None, False, 22, False) if not _status: if operation == "install": _msg += "DONE OK.\n" else: _msg += compare_versions(depkey, depsdict, _result_stdout.strip()) else: _msg += "NOT OK (exit code " + str(_status) + "). " if _msg.count("NOT OK"): _status = 701 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status) _result_stderr = str(obj.msg) _msg += "NOT OK (PMgr Exception, exit code " + str(_status) + "). "
def reset(global_objects, soft=True): ''' TBD ''' try: _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] _username = global_objects["space"]["username"] _logstore_username = global_objects["logstore"]["username"] _filestore_username = global_objects["filestore"]["username"] _filestore_config_file_fn = _stores_wk_dir + '/' + _filestore_username + "_rsync.conf" _msg = "Killing all processes..." print _msg, _proc_man = ProcessManagement() _proc_man.run_os_command("pkill -9 -u " + _username + " -f cbact") _proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-api") _proc_man.run_os_command("pkill -9 -u " + _username + " -f cloud-gui") _proc_man.run_os_command("pkill -9 -u " + _username + " -f ai-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f vm-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f submit-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f capture-") _proc_man.run_os_command("pkill -9 -u " + _username + " -f gmetad.py") _proc_man.run_os_command("pkill -9 -u " + _username + " -f gtkCBUI_") print "done" _proc_man.run_os_command("screen -wipe") _proc_man.run_os_command("rm -rf /tmp/restart_cb*" + global_objects["logstore"]["username"]) _proc_man.run_os_command("rm -rf /tmp/" + _username + "_*-*-*-*-*_avg_acc") _msg = "Flushing Object Store..." print _msg, _rmc = RedisMgdConn(global_objects["objectstore"]) _rmc.flush_object_store() print "done" _msg = "Flushing Log Store..." print _msg, _proc_man.run_os_command("pkill -9 -u " + _logstore_username + " -f rsyslogd") _file_list = [] _file_list.append("operations.log") _file_list.append("report.log") _file_list.append("submmiter.log") _file_list.append("loadmanager.log") _file_list.append("gui.log") _file_list.append("remotescripts.log") _file_list.append("messages.log") _file_list.append("monitor.log") _file_list.append("subscribe.log") for _fn in _file_list: _proc_man.run_os_command("rm -rf " + _log_dir + '/' + _logstore_username + '_' + _fn) _proc_man.run_os_command("touch " + _log_dir + '/' + _logstore_username + '_' + _fn) _status, _msg = syslog_logstore_setup(global_objects, "check") global_objects["logstore"]["just_restarted"] = True print "done\n" #_msg = "Flushing File Store..." #print _msg, #_proc_man.run_os_command("sudo pkill -9 -u root -f \"rsync --daemon --config " + _filestore_config_file_fn +"\"", raise_exception = False) #_proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _filestore_username + "_rsyncd.pid") #_status, _msg = rsync_filestore_setup(global_objects, "check") #print "done" if not soft: _msg = "Flushing Metric Store..." print _msg, _mmc = MongodbMgdConn(global_objects["metricstore"]) _mmc.flush_metric_store(global_objects["mon_defaults"]["username"]) print "done" _msg = "" _status = 0 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status) _msg = str(obj.msg)
def vmdestroy(self, obj_attr_list): """ TBD """ try: _status = 100 _fmsg = "An error has occurred, but no error message was captured" if "prov_cloud_ip" in obj_attr_list: try: _space_attr_list = self.osci.get_object( obj_attr_list["cloud_name"], "GLOBAL", False, "space", False ) _data_dir = _space_attr_list["data_working_dir"] + "/" + obj_attr_list["experiment_id"] if not access(_data_dir, F_OK): makedirs(_data_dir) _cmd = "scp -i " _cmd += _space_attr_list["ssh_key_name"] _cmd += " " + obj_attr_list["login"] + "@" + obj_attr_list["prov_cloud_ip"] _cmd += ":/home/" + obj_attr_list["login"] + "/nmon/nmon.csv " _cmd += _data_dir + "/nmon_" + obj_attr_list["cloud_vm_name"] + ".csv" print _cmd _proc_man = ProcessManagement() _proc_man.run_os_command(_cmd) except Exception, msg: cbdebug(str(msg), True) _time_mark_drs = int(time()) if "mgt_901_deprovisioning_request_originated" not in obj_attr_list: obj_attr_list["mgt_901_deprovisioning_request_originated"] = _time_mark_drs if not self.cskconn: self.connect(obj_attr_list["access"], obj_attr_list["api_key"], obj_attr_list["secret_key"]) _zoneid = self.get_zone(obj_attr_list["vmc_name"]) _wait = int(obj_attr_list["update_frequency"]) _instance = self.get_vm_instance(obj_attr_list) obj_attr_list["mgt_902_deprovisioning_request_sent"] = _time_mark_drs - int( obj_attr_list["mgt_901_deprovisioning_request_originated"] ) if _instance: _msg = "Sending a termination request for " + obj_attr_list["name"] + "" _msg += " (instance id " + _instance[u"id"] + ")" _msg += "...." cbdebug(_msg, True) self.cskconn.destroyVirtualMachine({u"id": _instance[u"id"]}) sleep(_wait) while self.is_vm_running(obj_attr_list): sleep(_wait) _time_mark_drc = int(time()) obj_attr_list["mgt_903_deprovisioning_request_completed"] = _time_mark_drc - _time_mark_drs else: True # This needs to be changed later. I could not find an easy way to # find the actual volume id of a given instance _volumes = self.cskconn.listVolumes({u"zoneid": _zoneid}) if _volumes and len(_volumes): for unattachedvol in _volumes: if unattachedvol[u"state"] == u"Allocated": _msg = unattachedvol[u"id"] + " " + unattachedvol[u"state"] _msg += "... was deleted" cbdebug(_msg) self.cskconn.deleteVolume({u"id": unattachedvol[u"id"]}) else: _msg = unattachedvol[u"id"] + " " + unattachedvol[u"state"] _msg += "... still attached and could not be deleted" cbdebug(_msg) else: _msg = "No volumes to remove" cbdebug(_msg) _status = 0
else : _msg = "##### File \"" + _file + "\" IGNORED...." cbwarn(_msg) if not len(depsdict) : _msg = "##### None of the files on the list \"" + str(_file_name_list) _msg += "\" contained configuration statements" cberr(_msg) exit(9) if _cleanup_repos : if not process_manager : process_manager = ProcessManagement(hostname) process_manager.run_os_command("sudo rm -rf /tmp/repoupdated", False) return True def docker_file_parser(depsdict, username, options, hostname, process_manager = False) : ''' TBD ''' _workloads_list = [] if options.role.count("orchestrator") : _workloads_list = [ "orchestrator", "orchprereqs" ] _role_list = [ options.role, "orchprereqs" ] if len(options.wks) > 1 : _workloads_list = options.wks.split(',')
def execute_command(operation, depkey, depsdict, hostname="127.0.0.1", username=None, process_manager=None): ''' TBD ''' try: _status = 100 _msg = "Obtaining command to be executed...." _cmd = {} _cmd["configure"] = get_cmdline(depkey, depsdict, "configure") _cmd["install"] = get_cmdline(depkey, depsdict, "install") if not process_manager: process_manager = ProcessManagement(hostname) _order = depsdict[depkey + "-order"] if depkey != "sudo" and operation == "configure": _msg = "(" + _order + ") Checking \"" + depkey + "\" version by executing the command \"" _msg += _cmd[operation] + "\"..." elif depkey == "sudo" and operation == "configure": _msg = "(" + _order + ") Checking passwordless sudo for the user \"" + username + "\" " _msg += "by executing the command \"" + _cmd[operation] + "\"..." else: _msg = "(" + _order + ") Installing \"" + depkey + "\" by executing the command \"" _msg += _cmd[operation] + "\"..." print _msg _msg = "RESULT: " if depkey == "repo" and operation == "install": build_repository_files(depsdict) _status, _result_stdout, _result_stderr = process_manager.run_os_command( _cmd[operation], False) if not _status: if operation == "install": _msg += "DONE OK.\n" else: _msg += compare_versions(depkey, depsdict, _result_stdout.strip()) else: print _result_stderr _msg += "NOT OK. " if _msg.count("NOT OK"): _status = 701 except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status) _result_stderr = str(obj.msg) _msg += "NOT OK. "
def reset(global_objects, soft = True) : ''' TBD ''' try : _msg = "Killing all processes..." print _msg, _proc_man = ProcessManagement() _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f cbact") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f cloud-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f ai-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f vm-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f submit-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f capture-") _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f gmetad.py") print "done" _proc_man.run_os_command("screen -wipe") _msg = "Flushing Object Store..." print _msg, _rmc = RedisMgdConn(global_objects["objectstore"]) _rmc.flush_object_store() print "done" if not soft : _msg = "Flushing Metric Store..." print _msg, _mmc = MongodbMgdConn(global_objects["metricstore"]) _mmc.flush_metric_store(global_objects["mon_defaults"]["username"]) print "done" _msg = "Flushing Log Store..." print _msg, _proc_man.run_os_command("pkill -9 -u " + global_objects["space"]["username"] + " -f rsyslogd") _proc_man.run_os_command("rm " + global_objects["space"]["stores_working_dir"] + "/logs/*.log") _status, _msg = syslog_logstore_setup(global_objects, "check") print "done" _msg = "" _status = 0 except ProcessManagement.ProcessManagementException, obj : _status = str(obj.status) _msg = str(obj.msg)
def rsync_filestore_setup(global_objects, operation="check"): ''' TBD ''' _hostname = global_objects["filestore"]["hostname"] _protocol = global_objects["filestore"]["protocol"] _username = global_objects["filestore"]["username"] _port = global_objects["filestore"]["port"] _usage = global_objects["filestore"]["usage"].lower() _base_dir = global_objects["space"]["base_dir"] _stores_wk_dir = global_objects["space"]["stores_working_dir"] _log_dir = global_objects["space"]["log_dir"] try: _name, _ip = hostname2ip(_hostname) if operation == "check": if _usage == "shared": _hostport = int(global_objects["filestore"]["port"]) if not pre_check_port(_hostname, _hostport, _protocol): _proc_man = ProcessManagement(username="******") _rsync_pid = _proc_man.get_pid_from_cmdline( "rsync --daemon") _cmd = "rsync --daemon" if not _rsync_pid: _msg = "Unable to detect a shared rsync server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _usage = "private" _proc_man = ProcessManagement(username=_username) _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf" _cmd = "rsync --daemon --config " + _config_file_fn if not access(_config_file_fn, F_OK): # File was deleted, but the rsync process is still dangling _proc_man.run_os_command("sudo pkill -9 -f " + _config_file_fn) _rsyslog_pid = _proc_man.get_pid_from_cmdline(_cmd) if not _rsyslog_pid: _proc_man.run_os_command("sudo rm -rf " + _stores_wk_dir + '/' + _username + "_rsyncd.pid") global_objects["filestore"][ "port"] = _proc_man.get_free_port( global_objects["filestore"]["port"], protocol="tcp") _hostport = int(global_objects["filestore"]["port"]) _config_file_contents = global_objects["filestore"][ "config_string"].replace('_', ' ') _config_file_contents = _config_file_contents.replace( "DOLLAR", '$') _config_file_contents = _config_file_contents.replace( "REPLEQUAL", '=') _config_file_contents = _config_file_contents.replace( "REPLPORT", str(_hostport)) _config_file_contents = _config_file_contents.replace( "REPLLOGDIR", _log_dir) _config_file_contents = _config_file_contents.replace( "REPLUSERU", _username + '_') _config_file_contents = _config_file_contents.replace( "REPLUSER", _username) _config_file_contents = _config_file_contents.replace( "REPLBASEDIR", _base_dir) _config_file_contents = _config_file_contents.replace( "REPLSTORESWORKINGDIR", global_objects["space"]["stores_working_dir"]) _config_file_contents = _config_file_contents.replace( ';', '\n') _config_file_contents = _config_file_contents.replace( "--", ';') _config_file_fn = _stores_wk_dir + '/' + _username + "_rsync.conf" _config_file_fd = open(_config_file_fn, 'w') _config_file_fd.write(_config_file_contents) _config_file_fd.close() _rsyslog_pid = _proc_man.start_daemon("sudo " + _cmd) if not _rsyslog_pid: _msg = "Unable to detect a private rsyslog server daemon running. " _msg += "Please try to start one (e.g., " + _cmd + ")" print _msg exit(8) else: _config_file_fd = open(_config_file_fn, 'r') _config_file_contents = _config_file_fd.readlines() _config_file_fd.close() for _line in _config_file_contents: if _line.count("port="): global_objects["filestore"]["port"] = _line.split( '=')[1] _hostport = int( global_objects["filestore"]["port"]) break _nh_conn = Nethashget(_hostname) _nh_conn.nmap(_hostport, _protocol) _msg = "A File Store of the kind \"rsync\" (" + _usage + ") " _msg += "on node " + _hostname + ", " + _protocol _msg += " port " + str(_hostport) + " seems to be running." cbdebug(_msg) _status = 0 return _status, _msg except socket.herror: _status = 1200 _msg = "The IP address \"" + _hostname + "\" - used by the rsync " _msg += " daemon - is not mapped to a Hostname. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except socket.gaierror: _status = 1200 _msg = "The Hostname \"" + _hostname + "\" - used by the rsync" _msg += " daemon - is not mapped to an IP. " _msg += "Please make sure this name is resolvable either in /etc/hosts or DNS." raise StoreSetupException(_msg, 9) except ProcessManagement.ProcessManagementException, obj: _status = str(obj.status) _msg = str(obj.msg) raise StoreSetupException(_msg, 9)
def instance_preparation(hostname, depsdict, options) : ''' TBD ''' try : _status = 100 _store = False _process_manager = False _cleanup = False _fmsg = "An error has occurred, but no error message was captured" if "stores" in depsdict : for _store in depsdict["stores"] : _process_manager = ProcessManagement(hostname) _cmd = depsdict[_store + "_command_check"] _msg = "Checking accesss from this instance to the CB" _msg += " Orchestrator's " + _store.capitalize() + " with" _msg += " \"" + _cmd + "\"..." cbinfo(_msg) _status, _x, _y = _process_manager.run_os_command(_cmd) _msg = "This instance was able to access CB Orchestrator's " + _store.capitalize() _msg += " with \"" + _cmd + "\" (OK)" cbinfo(_msg) if _store.lower() == "filestore" : print('\n') _msg = "rsync rsync://" + depsdict["Filestore_ip"] + ':' + depsdict["Filestore_port"] + '/' + depsdict["Filestore_username"] + "_cb" print(_msg) print('\n') _msg = "--filestore " + depsdict["Filestore_ip"] + '-' + depsdict["Filestore_port"] + '-' + depsdict["Filestore_username"] print(_msg) print('\n') if str(options.addr) != "bypass" : _cmd = options.wksdir + "/common/cb_cleanup.sh" _msg = "Running the instance cleanup script \"" + _cmd + "\"..." cbinfo(_msg) _process_manager.run_os_command(_cmd) _cleanup = True _status = 0 except Exception as e : _status = 23 _fmsg = str(e) finally : if _status : if _store and not _cleanup : _msg = "ERROR while checking accesss from this instance to the CB" _msg += " Orchestrator's " + _store.capitalize() + ": " + _fmsg _msg += " (make sure that the " + depsdict[_store + "_protocol"] _msg += " port " + depsdict[_store + "_port"] _msg += " is open at IP address " + depsdict[_store + "_ip"] if not _store and not _cleanup : _msg = "ERROR while preparing to check access from this instance " _msg += "to the CB Orchestrator's stores:" + _fmsg if _store and _cleanup : _msg = "ERROR while cleaning up instance." cberr(_msg) else : _msg = '' if _process_manager : _msg += "Sucessfully checked this instance's ability to connect" _msg += " to the Orchestrator's stores." if _cleanup and str(options.addr).lower() != "bypass" : _msg += "Sucessfully cleaned up this instance." cbinfo(_msg) return _status, _msg
def wait_for_instance_boot(self, obj_attr_list, time_mark_prc) : ''' TBD ''' _max_tries = int(obj_attr_list["update_attempts"]) _wait = int(obj_attr_list["update_frequency"]) _network_reachable = False _curr_tries = 0 if not _network_reachable : _msg = "Trying to establish network connectivity to " _msg += obj_attr_list["name"] + " (cloud-assigned uuid " _msg += obj_attr_list["cloud_vm_uuid"] + "), on IP address " _msg += obj_attr_list["prov_cloud_ip"] if str(obj_attr_list["use_jumphost"]).lower() == "false" : _msg += "..." else : _msg += " via jumphost " + obj_attr_list["jumphost_ip"] + "..." obj_attr_list["check_boot_complete"] = "run_command_/bin/true" cbdebug(_msg, True) self.pending_set(obj_attr_list, _msg) sleep(_wait) while not _network_reachable and _curr_tries < _max_tries : _start_pooling = int(time()) if "async" not in obj_attr_list or str(obj_attr_list["async"]).lower() == "false" : if threading.current_thread().abort : _msg = "VM Create Aborting..." _status = 123 raise CldOpsException(_msg, _status) if obj_attr_list["check_boot_complete"].count("tcp_on_") : _nh_conn = Nethashget(obj_attr_list["prov_cloud_ip"]) _port_to_check = obj_attr_list["check_boot_complete"].replace("tcp_on_",'') _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish a TCP connection to port " _msg += str(_port_to_check) + " on address " _msg += obj_attr_list["prov_cloud_ip"] cbdebug(_msg) _vm_is_booted = _nh_conn.check_port(int(_port_to_check), "TCP") elif obj_attr_list["check_boot_complete"].count("cloud_ping") : _msg = "Check if the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted by " _msg += "attempting to establish network connectivity " _msg += "through the cloud's API" cbdebug(_msg) _vm_is_booted = self.is_vm_alive(obj_attr_list) elif obj_attr_list["check_boot_complete"].count("subscribe_on_") : _string_to_search = obj_attr_list["prov_cloud_ip"] + " is " _string_to_search += "booted" _channel_to_subscribe = obj_attr_list["check_boot_complete"].replace("subscribe_on_",'') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list["cloud_vm_uuid"] + ") has booted by " _msg += "subscribing to channel \"" + str(_channel_to_subscribe) _msg += "\" and waiting for the message \"" _msg += _string_to_search + "\"." cbdebug(_msg) self.osci.add_to_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) _sub_channel = self.osci.subscribe(obj_attr_list["cloud_name"], "VM", _channel_to_subscribe, _max_tries * _wait) for _message in _sub_channel.listen() : if str(_message["data"]).count(_string_to_search) : _vm_is_booted = True break _sub_channel.unsubscribe() self.osci.remove_from_list(obj_attr_list["cloud_name"], "VM", "VMS_BOOTING", obj_attr_list["prov_cloud_ip"]) elif obj_attr_list["check_boot_complete"].count("wait_for_") : _boot_wait_time = int(obj_attr_list["check_boot_complete"].replace("wait_for_",'')) _msg = "Assuming that the VM \"" + obj_attr_list["cloud_name"] _msg += "\" (" + obj_attr_list["name"] + ") is booted after" _msg += " waiting for " + str(_boot_wait_time) + " seconds." cbdebug(_msg) if _boot_wait_time : sleep(_boot_wait_time) _vm_is_booted = True elif obj_attr_list["check_boot_complete"].count("run_command_") : _command_to_run = obj_attr_list["check_boot_complete"].replace("run_command_",'') _command_to_run = _command_to_run.replace("____",' ') _msg = "Check if the VM \"" + obj_attr_list["name"] _msg += "\" (" + obj_attr_list["cloud_vm_uuid"] + ") has booted by " _msg += "running the command \"" + str(_command_to_run) cbdebug(_msg) if _curr_tries <= _max_tries/3 : _connection_timeout = int(obj_attr_list["update_frequency"])/2 elif _curr_tries > _max_tries/3 and _curr_tries < 2*_max_tries/3 : _connection_timeout = int(obj_attr_list["update_frequency"]) obj_attr_list["comments"] += "Had to increase ssh timeout. " else : _connection_timeout = int(obj_attr_list["update_frequency"])*2 obj_attr_list["comments"] += "Had to increase ssh timeout one more time. " if str(obj_attr_list["use_jumphost"]).lower() == "true" : if "ssh_config_file" in obj_attr_list: _ssh_conf_file = obj_attr_list["ssh_config_file"] else: _ssh_conf_file = None else : _ssh_conf_file = None _proc_man = ProcessManagement(username = obj_attr_list["login"], \ cloud_name = obj_attr_list["cloud_name"], \ hostname = obj_attr_list["prov_cloud_ip"], \ priv_key = obj_attr_list["identity"], \ config_file = _ssh_conf_file, connection_timeout = _connection_timeout) try : _status, _result_stdout, _result_stderr = _proc_man.run_os_command(_command_to_run) if not _status : _vm_is_booted = True else : _vm_is_booted = False except : _vm_is_booted = False elif obj_attr_list["check_boot_complete"].count("snmpget_poll") : import netsnmp # Send SNMP GET message. Flag VM as booted if any response at all is recieved _vm_is_booted = False try : _msg = "Opening SNMP session to " + obj_attr_list["cloud_ip"] cbdebug(_msg) _snmp_wait_time = _wait * 1000000 _snmp_version = int(obj_attr_list["snmp_version"]) _snmp_comm = str(obj_attr_list["snmp_community"]) _snmp_session = netsnmp.Session(Version=_snmp_version, \ DestHost=obj_attr_list["cloud_ip"], \ Community=_snmp_comm, \ Timeout=_snmp_wait_time, Retries=0) _vars = netsnmp.VarList(netsnmp.Varbind(obj_attr_list["snmp_variable"], '0')) _snmp_response = _snmp_session.get(_vars) except : if _snmp_session.ErrorStr : _msg = "Error in SNMP handler : " + _snmp_session.ErrorStr else : _msg = "Unknown error in SNMP handler." cbdebug(_msg) _status = 200 raise CldOpsException(_msg, _status) if (_snmp_response[0] != None ) : _vm_is_booted = True _msg = "SNMP Response: " + str(_snmp_response) cbdebug(_msg) else : _vm_is_booted = False _msg = "Warning: No valid method specified to determined if VM has booted." cbdebug(_msg, True) _pooling_time = int(time()) - _start_pooling if _pooling_time <= _wait : _actual_wait = _wait - _pooling_time else : _msg = "The time spent on pooling for \"booted\" status (" + str(_pooling_time) _msg += " s) is actually longer than the " _msg += "interval between pooling attempts (" + str(_wait) + " s)." cbdebug(_msg, True) _actual_wait = 0 if _vm_is_booted : obj_attr_list["mgt_004_network_acessible"] = int(time()) - time_mark_prc self.pending_set(obj_attr_list, "Network accessible now. Continuing...") _network_reachable = True break else : _msg = "(" + str(_curr_tries) + ") " + obj_attr_list["name"] _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") " _msg += "still not network reachable. Will wait for " + str(_actual_wait) _msg += " seconds and check again." self.pending_set(obj_attr_list, _msg) cbdebug(_msg) sleep(_actual_wait) _curr_tries += 1 if _curr_tries < _max_tries : _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") " _msg += "is network reachable (boot process finished successfully)" cbdebug(_msg) obj_attr_list["arrival"] = int(time()) # It should be mgt_006 and mgt_007 NOT mgt_005 obj_attr_list["mgt_006_instance_preparation"] = "0" obj_attr_list["mgt_007_application_start"] = "0" self.pending_set(obj_attr_list, "Application starting up...") self.get_attr_from_pending(obj_attr_list, "all") else : _msg = "" + obj_attr_list["name"] + "" _msg += " (cloud-assigned uuid " + obj_attr_list["cloud_vm_uuid"] + ") " _msg += "is not network reachable after " + str(_max_tries * _wait) + " seconds.... " _msg += "Giving up." cberr(_msg, True) raise CldOpsException(_msg, 89)
def expand_command(cmdline, depsdict, process_manager=False): ''' TBD ''' _actual_cmdline = '' for _command in cmdline.split(';'): if _command.count("package_install"): _packages = _command.replace("package_install", '') if depsdict["cdistkind"] == "ubuntu": if _packages.count(".deb"): _command = "sudo dpkg -i PACKAGES; sudo apt-get -f install -y --force-yes --allow-unauthenticated " else: _command = "sudo apt-get -q -y --force-yes --allow-unauthenticated -o Dpkg::Options::=\"--force-confnew\" install PACKAGES" elif depsdict["cdistkind"] == "rhel" or depsdict[ "cdistkind"] == "fedora": if _packages.count(".rpm"): _command = "sudo rpm -i PACKAGES" else: _command = "sudo yum -y install PACKAGES" _command = _command.replace("PACKAGES", _packages) if _command.count("service_restart_enable") or _command.count( "service_stop_disable"): if not process_manager: process_manager = ProcessManagement() if _command.count("service_restart_enable"): _services = _command.replace("service_restart_enable", '') if _command.count("service_stop_disable"): _services = _command.replace("service_stop_disable", '') _command = '' for _service in _services.split(): _status, _result_stdout, _result_stderr = \ process_manager.run_os_command("sudo systemctl status " + _service, \ raise_exception = False) if not _status: if cmdline.count("service_restart_enable"): _service_cmd = "sudo systemctl restart SERVICE; sudo systemctl enable SERVICE" if cmdline.count("service_stop_disable"): _service_cmd = "sudo systemctl stop SERVICE; sudo systemctl disable SERVICE" else: if cmdline.count("service_restart_enable"): _service_cmd = "sudo service SERVICE restart;" if depsdict["cdistkind"] == "rhel" or depsdict[ "cdistkind"] == "fedora": _service_cmd += "sudo chkconfig SERVICE on;" elif depsdict["cdistkind"] == "ubuntu": if os.access("/etc/init/" + _service + ".conf", os.F_OK): _service_cmd += "sudo rm -rf /etc/init/SERVICE.override" else: _service_cmd += "sudo update-rc.d -f SERVICE defaults" if cmdline.count("service_stop_disable"): _service_cmd = "sudo systemctl SERVICE stop; sudo chkconfig SERVICE off;" if depsdict["cdistkind"] == "rhel" or depsdict[ "cdistkind"] == "fedora": _service_cmd = "sudo chkconfig SERVICE off;" elif depsdict["cdistkind"] == "ubuntu": if os.access("/etc/init/" + _service + ".conf", os.F_OK): _service_cmd = "sudo sh -c 'echo manual > /etc/init/SERVICE.override'" else: _service_cmd = "sudo update-rc.d -f SERVICE remove" _service_cmd = _service_cmd.replace("SERVICE", _service) + ';' _command += _service_cmd _actual_cmdline += _command + ';' return _actual_cmdline
else : _msg = "File \"" + _file + "\" IGNORED...." print _msg if not len(depsdict) : _msg = "Error: None of the files on the list \"" + str(_file_name_list) _msg += "\" contained configuration statements" print _msg exit(9) if _cleanup_repos : if not process_manager : process_manager = ProcessManagement(hostname) process_manager.run_os_command("sudo rm -rf /tmp/repoupdated", False) return True def get_linux_distro() : ''' TBD ''' _linux_distro_name, _linux_distro_ver, _x = platform.linux_distribution() if _linux_distro_name.count("Red Hat") : _distro = "rhel" elif _linux_distro_name.count("Scientific Linux") : _distro = "rhel" elif _linux_distro_name.count("CentOS") : _distro = "rhel" elif _linux_distro_name.count("Ubuntu") :
else: _msg = "##### File \"" + _file + "\" IGNORED...." cbwarn(_msg) if not len(depsdict): _msg = "##### None of the files on the list \"" + str(_file_name_list) _msg += "\" contained configuration statements" cberr(_msg) exit(9) if _cleanup_repos: if not process_manager: process_manager = ProcessManagement(hostname) process_manager.run_os_command("sudo rm -rf /tmp/repoupdated", False) return True def docker_file_parser(depsdict, username, options, hostname, process_manager=False): ''' TBD ''' _workloads_list = [] if options.role.count("orchestrator"):