def select_url(source, depsdict):
'''
TBD
'''
depsdict[source + "_addr_list"] = []
depsdict[source + "_addr"] = False
if source == "repo":
_element = "package repository"
else:
_element = "python pip repository"
print '\n'
_msg = "Selecting " + _element + " address...."
cbinfo(_msg)
for _key in sorted(depsdict.keys()):
if _key.count(source + "-addr"):
_index = int(_key.replace(source + "-addr", ''))
depsdict[source + "_addr_list"].insert(_index, depsdict[_key])
for _repo_addr in depsdict[source + "_addr_list"]:
if check_url("http://" + _repo_addr, "ARCH", depsdict["carch"]):
depsdict[source + "_addr"] = _repo_addr
if len(depsdict[source + "_addr_list"]):
if depsdict[source + "_addr"]:
_msg = "A " + _element + " in \"" + depsdict[
source + "_addr"] + "\" seems to be up"
depsdict[source +
"_dropbox"] = "http://" + depsdict[source +
"_addr"] + "/dropbox"
depsdict[source + "_credentials_url"] = "http://" + depsdict[
source + "_addr"] + "/dropbox/ssh_keys"
cbinfo(_msg)
else:
_msg = "##### None of the indicated " + _element + " was available. ".replace(
"repository", "repositories")
if source == "repo":
_msg += "Will ignore any repository URL that has the keyword REPO_ADDR..."
cbwarn(_msg)
else:
_msg = "##### No " + _element + " specified. ".replace(
"repository", "repositories")
if source == "repo":
_msg += "Will ignore any repository URL that has the keyword REPO_ADDR..."
cbwarn(_msg)
return True
def select_url(source, depsdict) :
'''
TBD
'''
depsdict[source + "_addr_list"] = []
depsdict[source + "_addr"] = False
if source == "repo" :
_element = "package repository"
else :
_element = "python pip repository"
print '\n'
_msg = "Selecting " + _element + " address...."
cbinfo(_msg)
for _key in sorted(depsdict.keys()) :
if _key.count(source + "-addr") :
_index = int(_key.replace(source + "-addr",''))
depsdict[source + "_addr_list"].insert(_index, depsdict[_key])
for _repo_addr in depsdict[source + "_addr_list"] :
if check_url("http://" + _repo_addr, "ARCH", depsdict["carch"]) :
depsdict[source + "_addr"] = _repo_addr
if len(depsdict[source + "_addr_list"]) :
if depsdict[source + "_addr"] :
_msg = "A " + _element + " in \"" + depsdict[source + "_addr"] + "\" seems to be up"
depsdict[source + "_dropbox"] = "http://" + depsdict[source + "_addr"] + "/dropbox"
depsdict[source + "_credentials_url"] = "http://" + depsdict[source + "_addr"] + "/dropbox/ssh_keys"
cbinfo(_msg)
else :
_msg = "##### None of the indicated " + _element + " was available. ".replace("repository","repositories")
if source == "repo" :
_msg += "Will ignore any repository URL that has the keyword REPO_ADDR..."
cbwarn(_msg)
else :
_msg = "##### No " + _element + " specified. ".replace("repository","repositories")
if source == "repo" :
_msg += "Will ignore any repository URL that has the keyword REPO_ADDR..."
cbwarn(_msg)
return True
def preparation_file_parser(depsdict,
username,
options,
hostname,
process_manager=False):
'''
TBD
'''
_file = home + "/cb_prepare_parameters.txt"
print '\n'
if os.access(_file, os.F_OK):
_msg = "##### Parsing CB file \"" + _file + "\"...."
cbinfo(_msg)
_fd = open(_file, 'r')
_fc = _fd.readlines()
_fd.close()
if "stores" not in depsdict:
depsdict["stores"] = []
for _line in _fc:
_line = _line.strip()
_store, _store_ip, _store_port, _store_protocol, _store_username = _line.split(
)
_cmd = "nc -z -w 3 -" + _store_protocol[0].lower(
) + ' ' + _store_ip + ' ' + _store_port
if _store not in depsdict["stores"]:
depsdict["stores"].append(_store)
depsdict[_store + "_ip"] = _store_ip
depsdict[_store + "_port"] = _store_port
depsdict[_store + "_protocol"] = _store_protocol
depsdict[_store + "_username"] = _store_username
depsdict[_store + "_command_check"] = _cmd
return True
def check_port(self, port = None, protocol = "TCP") :
'''
TBD
'''
try :
if protocol == "TCP" :
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
elif protocol == "UDP" :
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.settimeout(5)
self.socket.connect((self.hostname, self.port if port is None else port))
return True
except socket.error, msg :
_msg = "Unable to connect to " + protocol + " port " + str(port)
_msg += " on host " + self.hostname + ": " + str(msg)
cbinfo(_msg)
return False
self.socket.close()
self.socket = None
def preparation_file_parser(depsdict, username, options, hostname, process_manager = False) :
'''
TBD
'''
_file = home + "/cb_prepare_parameters.txt"
print '\n'
if os.access(_file, os.F_OK) :
_msg = "##### Parsing CB file \"" + _file + "\"...."
cbinfo(_msg)
_fd = open(_file, 'r')
_fc = _fd.readlines()
_fd.close()
if "stores" not in depsdict :
depsdict["stores"] = []
for _line in _fc :
_line = _line.strip()
_store, _store_ip, _store_port, _store_protocol, _store_username = _line.split()
_cmd = "nc -z -w 3 -" + _store_protocol[0].lower() + ' ' + _store_ip + ' ' + _store_port
if _store not in depsdict["stores"] :
depsdict["stores"].append(_store)
depsdict[_store + "_ip"] = _store_ip
depsdict[_store + "_port"] = _store_port
depsdict[_store + "_protocol"] = _store_protocol
depsdict[_store + "_username"] = _store_username
depsdict[_store + "_command_check"] = _cmd
return True
def instance_preparation(hostname, depsdict, options) :
'''
TBD
'''
try :
_status = 100
_store = False
_process_manager = False
_cleanup = False
_fmsg = "An error has occurred, but no error message was captured"
if "stores" in depsdict :
for _store in depsdict["stores"] :
_process_manager = ProcessManagement(hostname)
_cmd = depsdict[_store + "_command_check"]
_msg = "Checking accesss from this instance to the CB"
_msg += " Orchestrator's " + _store.capitalize() + " with"
_msg += " \"" + _cmd + "\"..."
cbinfo(_msg)
_status, _x, _y = _process_manager.run_os_command(_cmd)
_msg = "This instance was able to access CB Orchestrator's " + _store.capitalize()
_msg += " with \"" + _cmd + "\" (OK)"
cbinfo(_msg)
if _store.lower() == "filestore" :
print '\n'
_msg = "rsync rsync://" + depsdict["Filestore_ip"] + ':' + depsdict["Filestore_port"] + '/' + depsdict["Filestore_username"] + "_cb"
print _msg
print '\n'
_msg = "--filestore " + depsdict["Filestore_ip"] + '-' + depsdict["Filestore_port"] + '-' + depsdict["Filestore_username"]
print _msg
print '\n'
if str(options.addr) != "bypass" :
_cmd = options.wksdir + "/common/cb_cleanup.sh"
_msg = "Running the instance cleanup script \"" + _cmd + "\"..."
cbinfo(_msg)
_process_manager.run_os_command(_cmd)
_cleanup = True
_status = 0
except Exception, e :
_status = 23
_fmsg = str(e)
def instance_preparation(hostname, depsdict, options) :
'''
TBD
'''
try :
_status = 100
_store = False
_process_manager = False
_cleanup = False
_fmsg = "An error has occurred, but no error message was captured"
if "stores" in depsdict :
for _store in depsdict["stores"] :
_process_manager = ProcessManagement(hostname)
_cmd = depsdict[_store + "_command_check"]
_msg = "Checking accesss from this instance to the CB"
_msg += " Orchestrator's " + _store.capitalize() + " with"
_msg += " \"" + _cmd + "\"..."
cbinfo(_msg)
_status, _x, _y = _process_manager.run_os_command(_cmd)
_msg = "This instance was able to access CB Orchestrator's " + _store.capitalize()
_msg += " with \"" + _cmd + "\" (OK)"
cbinfo(_msg)
if _store.lower() == "filestore" :
print '\n'
_msg = "rsync rsync://" + depsdict["Filestore_ip"] + ':' + depsdict["Filestore_port"] + '/' + depsdict["Filestore_username"] + "_cb"
print _msg
print '\n'
_msg = "--filestore " + depsdict["Filestore_ip"] + '-' + depsdict["Filestore_port"] + '-' + depsdict["Filestore_username"]
print _msg
print '\n'
if str(options.addr) != "bypass" :
_cmd = options.wksdir + "/common/cb_cleanup.sh"
_msg = "Running the instance cleanup script \"" + _cmd + "\"..."
cbinfo(_msg)
_process_manager.run_os_command(_cmd)
_cleanup = True
_status = 0
except Exception, e :
_status = 23
_fmsg = str(e)
def deps_file_parser(depsdict, username, options, hostname, process_manager = False) :
'''
TBD
'''
_file_name_list = []
_file_name_list.append(options.defdir + "/PUBLIC_dependencies.txt")
_cleanup_repos = False
if len(options.wks) > 1 :
_workloads_list = options.wks.split(',')
for _workload in _workloads_list :
_file_name_list.append(options.wksdir + '/' + _workload + "/dependencies.txt")
_cleanup_repos = True
_file_name_list.append(options.defdir + "/IBM_dependencies.txt")
_file_name_list.append(options.defdir + "/SPEC_dependencies.txt")
if len(options.custom) :
_file_name_list.append(options.cusdir + '/' + options.custom)
print '\n'
for _file in _file_name_list :
if os.access(_file, os.F_OK) :
try:
_fd = open(_file, 'r')
_fc = _fd.readlines()
_fd.close()
_msg = "##### File \"" + _file + "\" opened and loaded...."
cbinfo(_msg)
for _line in _fc :
_line = _line.strip()
if _line.count("#",0,2) :
_sstr = None
elif len(_line) < 3 :
_sstr = None
elif _line.count(" = ") :
_sstr = " = "
elif _line.count(" =") :
_sstr = " ="
elif _line.count("= ") :
_sstr = "= "
elif _line.count("=") :
_sstr = "="
else :
_sstr = None
if _sstr :
_key, _value = _line.split(_sstr)
_key = _key.strip()
depsdict[_key] = _value
except Exception, e :
_msg = "##### Error reading file \"" + _file + "\":" + str(e)
cberr(_msg)
exit(4)
else :
_msg = "##### File \"" + _file + "\" IGNORED...."
cbwarn(_msg)
def docker_file_parser(depsdict, username, options, hostname, process_manager = False) :
'''
TBD
'''
_workloads_list = []
if options.role.count("orchestrator") :
_workloads_list = [ "orchestrator", "orchprereqs" ]
_role_list = [ options.role, "orchprereqs" ]
if len(options.wks) > 1 :
_workloads_list = options.wks.split(',')
_role_list = [ options.role ]
print '\n'
for _role in _role_list :
for _path, _dirs, _files in os.walk(options.dfdir + '/' + _role) :
for _fnam in _files:
_full_fnam = os.path.join(_path, _fnam)
if not _fnam.count("._processed_") and _fnam.count("Dockerfile-") and os.access(_full_fnam, os.F_OK) :
_x_fnam = _fnam.replace("Dockerfile-",'').split('_')
_key_prefix = _x_fnam[0]
_f_workload = _x_fnam[2]
if len(_x_fnam) > 3 :
_f_workload += '_' + _x_fnam[3]
if _f_workload in _workloads_list :
try:
_fd = open(_full_fnam, 'r')
_fc = _fd.readlines()
_fd.close()
_msg = "##### Parsing Dockerfile \"" + _full_fnam + "\"...."
cbinfo(_msg)
_current_key = None
for _line in _fc :
_line = _line.strip()
if _current_key :
if _line.count("#") and _line.count("-install-") :
depsdict[_current_key] = depsdict[_current_key][0:-1]
if _current_key.count("centos-") :
depsdict[_current_key.replace("centos-","rhel-")] = depsdict[_current_key]
depsdict[_current_key.replace("centos-","fedora-")] = depsdict[_current_key]
_current_key = None
elif _line.count("#") and _line.count("RUN") :
True
else :
_line = _line.replace("RUN apt-get install -y","package_install")
_line = _line.replace("RUN yum install -y","package_install")
_line = _line.replace("RUN pip install --upgrade", "sudo pip install --upgrade INDEXURL")
_line = _line.replace("RUN git", "git")
_line = _line.replace("; apt-get install", "; sudo apt-get install")
_line = _line.replace("; apt-get update", "; sudo apt-get update")
_line = _line.replace("; add-apt-repository", "; sudo add-apt-repository")
_line = _line.replace("; dpkg", "; sudo dpkg")
_line = _line.replace("; yum install", "; sudo yum install")
_line = _line.replace("; rpm", "; sudo rpm")
_line = _line.replace("; chown", "; sudo chown")
_line = _line.replace("; make install", "; sudo make install")
_line = _line.replace("RUN mkdir -p /home/REPLACE_USERNAME/cbtool/3rd_party", "mkdir -p 3RPARTYDIR")
_line = _line.replace("RUN ", "sudo ")
_line = _line.replace("ENV ", "export ")
_line = _line.replace("sudo cd ", "cd ")
_line = _line.replace("sudo export", "export ")
_line = _line.replace("sudo sudo ", "sudo ")
_line = _line.replace("sudo REPLACE_RSYNC", "REPLACE_RSYNC")
_line = _line.replace("WORKDIR /home/REPLACE_USERNAME/cbtool/3rd_party", "cd 3RPARTYDIR")
_line = _line.replace("# service_stop_disable", "service_stop_disable")
_line = _line.replace("# echo", "echo")
_line = _line.replace("/home/root", "/root")
_line = _line.replace("____",' ')
depsdict[_current_key] += _line + "; "
else :
if _line.count("#") and _line.count("-install-") :
_current_key = _key_prefix + '-' + _line.replace("#",'').strip()
depsdict[_current_key] = ''
except Exception, e :
_msg = "##### Error reading file \"" + _full_fnam + "\":" + str(e)
cberr(_msg)
exit(4)
def dependency_checker_installer(hostname, depsdict, username, operation, options) :
'''
TBD
'''
try :
_status = 100
_dep_missing = -1
_fmsg = "An error has occurred, but no error message was captured"
if len(options.wks) > 1 :
if options.wks.count("_ycsb") :
options.wks += ",ycsb"
if options.wks.count(",ycsb") :
options.wks += ",mongo_ycsb,cassandra_ycsb,redis_ycsb"
if options.wks.count(",acmeair") :
options.wks += ",mongo_acmeair"
deps_file_parser(depsdict, username, options, "127.0.0.1")
docker_file_parser(depsdict, username, options, "127.0.0.1")
preparation_file_parser(depsdict, username, options, "127.0.0.1")
if "Filestore_ip" not in depsdict :
depsdict["Filestore_ip"], depsdict["Filestore_port"], depsdict["Filestore_username"] = options.filestore.split('-')
depsdict["cdistkind"], depsdict["cdistver"], depsdict["cdistmajorver"], depsdict["cdistnam"], depsdict["carch"] = get_linux_distro()
depsdict["3rdpartydir"] = options.tpdir
depsdict["scriptsdir"] = options.wksdir
depsdict["credentialsdir"] = options.creddir
depsdict["username"] = username
if options.addr :
depsdict["repo-addr1"] = options.addr
depsdict["pip-addr1"] = options.addr
if depsdict["carch"] == "x86_64" :
depsdict["carch1"] = "x86_64"
depsdict["carch2"] = "x86-64"
depsdict["carch3"] = "amd64"
elif depsdict["carch"] == "ppc64le" :
depsdict["carch1"] = "ppc64le"
depsdict["carch2"] = "ppc64"
depsdict["carch3"] = "ppc64"
else:
depsdict["carch1"] = "aarch64"
depsdict["carch2"] = "aarch64"
depsdict["carch3"] = "aarch64"
_missing_dep = []
_dep_list = [0] * 5000
if str(options.addr) != "bypass" :
select_url("repo", depsdict)
select_url("pip", depsdict)
_raise_exception = True
else :
depsdict["pip_addr"] = None
depsdict["repo_addr"] = None
_raise_exception = False
for _key in depsdict.keys() :
if _key.count("-order") :
_dependency = _key.replace("-order",'')
_order = int(depsdict[_key]) * 20
_dep_list.insert(_order, _dependency)
_dep_list = [x for x in _dep_list if x != 0]
print '\n'
if options.role.count("workload") :
options.tag = "base," + options.role
_msg = "##### This node will be used to play a role in the Virtual Applications"
_msg += " (AIs) \"" + str(options.wks) + "\". Only a subset of the depedencies"
_msg += " will be " + operation + "ed. This node cannot be used as an Orchestrator Node\n"
_msg += "\n"
cbinfo(_msg)
_hadoop_helper = "#!/usr/bin/env bash\n"
_hadoop_helper += "for HADOOP_CPATH in ~ /usr/local\n"
_hadoop_helper += "do\n"
_hadoop_helper += " if [[ $(sudo ls $HADOOP_CPATH | grep -v tar | grep -v tgz | grep -v spark | grep -c hadoop) -ne 0 ]]\n"
_hadoop_helper += " then\n"
_hadoop_helper += " eval HADOOP_CPATH=${HADOOP_CPATH}\n"
_hadoop_helper += " HADOOP_HOME=$(ls ${HADOOP_CPATH} | grep -v tar | grep -v tgz | grep -v spark | grep -v hadoop_store | grep hadoop | sort -r | head -n1)\n"
_hadoop_helper += " eval HADOOP_HOME=\"$HADOOP_CPATH/${HADOOP_HOME}\"\n"
_hadoop_helper += " if [[ -d $HADOOP_HOME ]]\n"
_hadoop_helper += " then\n"
_hadoop_helper += " echo \"1\"\n"
_hadoop_helper += " exit 0\n"
_hadoop_helper += " fi\n"
_hadoop_helper += " fi\n"
_hadoop_helper += "done\n"
_hadoop_helper += "echo \"0\"\n"
_hadoop_helper += "exit 1\n"
try:
_file_name = "/tmp/get_hadoop"
_file_descriptor = file(_file_name, 'w')
_file_descriptor.write(_hadoop_helper)
_file_descriptor.close()
os.chmod(_file_name, 0755)
except IOError, msg :
_msg = "######## Error writing file \"" + _file_name + "\":" + str(msg)
cberr(_msg)
exit(4)
except OSError, msg :
_msg = "######## Error writing file \"" + _file_name + "\":" + str(msg)
cberr(_msg)
exit(4)
def dependency_checker_installer(hostname, depsdict, username, operation, options) :
'''
TBD
'''
try :
_status = 100
_dep_missing = -1
_fmsg = "An error has occurred, but no error message was captured"
if len(options.wks) > 1 :
if options.wks.count("_ycsb") :
options.wks += ",ycsb"
if options.wks.count(",ycsb") :
options.wks += ",mongo_ycsb,cassandra_ycsb,redis_ycsb"
if options.wks.count(",acmeair") :
options.wks += ",mongo_acmeair"
deps_file_parser(depsdict, username, options, "127.0.0.1")
docker_file_parser(depsdict, username, options, "127.0.0.1")
preparation_file_parser(depsdict, username, options, "127.0.0.1")
if "Filestore_ip" not in depsdict :
depsdict["Filestore_ip"], depsdict["Filestore_port"], depsdict["Filestore_username"] = options.filestore.split('-')
depsdict["cdistkind"], depsdict["cdistver"], depsdict["cdistmajorver"], depsdict["cdistnam"], depsdict["carch"] = get_linux_distro()
depsdict["3rdpartydir"] = options.tpdir
depsdict["scriptsdir"] = options.wksdir
depsdict["credentialsdir"] = options.creddir
depsdict["username"] = username
if options.addr :
depsdict["repo-addr1"] = options.addr
depsdict["pip-addr1"] = options.addr
if depsdict["carch"] == "x86_64" :
depsdict["carch1"] = "x86_64"
depsdict["carch2"] = "x86-64"
depsdict["carch3"] = "amd64"
elif depsdict["carch"] == "ppc64le" :
depsdict["carch1"] = "ppc64le"
depsdict["carch2"] = "ppc64"
depsdict["carch3"] = "ppc64"
else:
depsdict["carch1"] = "aarch64"
depsdict["carch2"] = "aarch64"
depsdict["carch3"] = "aarch64"
_missing_dep = []
_dep_list = [0] * 5000
if str(options.addr) != "bypass" :
select_url("repo", depsdict)
select_url("pip", depsdict)
_raise_exception = True
else :
depsdict["pip_addr"] = None
depsdict["repo_addr"] = None
_raise_exception = False
for _key in depsdict.keys() :
if _key.count("-order") :
_dependency = _key.replace("-order",'')
_order = int(depsdict[_key]) * 20
_dep_list.insert(_order, _dependency)
_dep_list = [x for x in _dep_list if x != 0]
print '\n'
if options.role.count("workload") :
options.tag = "base," + options.role
_msg = "##### This node will be used to play a role in the Virtual Applications"
_msg += " (AIs) \"" + str(options.wks) + "\". Only a subset of the depedencies"
_msg += " will be " + operation + "ed. This node cannot be used as an Orchestrator Node\n"
_msg += "\n"
cbinfo(_msg)
else :
options.tag = "base," + options.role + ',' + options.clouds
_msg = "##### This node will be prepared as an Orchestration Node."
_msg += " The full set of dependencies will be " + operation + "ed. "
_msg += "\n"
cbinfo(_msg)
options.tag = options.tag.split(',')
_selected_dep_list = []
for _dep in _dep_list :
for _tag in options.tag :
if _dep + "-tag" in depsdict :
_dep_tag_list = depsdict[_dep + "-tag"].split(',')
else :
_dep_tag_list = [ "workload" ]
if _tag in _dep_tag_list :
if _dep not in _selected_dep_list :
_selected_dep_list.append(_dep)
_dep_list = _selected_dep_list
_process_manager = ProcessManagement(hostname)
_status, _std_out, _y = _process_manager.run_os_command("sudo cat /proc/1/cgroup | grep -c docker", raise_exception = False)
if _status :
depsdict["indocker"] = False
else :
if str(_std_out.replace("\n",'')) == '0' :
depsdict["indocker"] = False
else :
depsdict["indocker"] = True
_msg = "##### DETECTED OPERATING SYSTEM KIND: " + depsdict["cdistkind"]
cbinfo(_msg)
_msg = "##### DETECTED OPERATING SYSTEM VERSION: " + depsdict["cdistver"] + " (" + depsdict["cdistmajorver"] + ')'
cbinfo(_msg)
_msg = "##### DETECTED OPERATING SYSTEM NAME: " + depsdict["cdistnam"]
cbinfo(_msg)
_msg = "##### DETECTED ARCHITECTURE: " + depsdict["carch"]
cbinfo(_msg)
_msg = "##### DETECTED RUNNING INSIDE DOCKER: " + str(depsdict["indocker"])
cbinfo(_msg)
print '\n'
if operation == "configure" :
if "repo" in _dep_list :
_dep_list.remove("repo")
if depsdict["cdistkind"] == "AMI" :
_msg = "This node runs the \"" + depsdict["cdistkind"] + "\" Linux "
_msg += "distribution. Will treat it as \"rhel\", but will disable"
_msg += " the repository manipulation."
cbinfo(_msg)
depsdict["cdistkind"] = "rhel"
if "repo" in _dep_list :
_dep_list.remove("repo")
if depsdict["carch"].count("ppc") and "mongdob" in _dep_list :
_msg = "##### The processors on this node have a \"Power\" architecture."
_msg += "Removing MongoDB and Chef (client) from the dependency list"
cbwarn(_msg)
_dep_list.remove("mongodb")
_dep_list.remove("chef-client")
if "java" in _dep_list and "oraclejava" in _dep_list :
_msg = "Since both \"java\" and \"oraclejava\" are listed as dependencies"
_msg += ", only \"oraclejava\" will be used"
cbinfo(_msg)
_dep_list.remove("java")
_dep_list.remove("java-home")
_fmsg = ""
_dep_missing = 0
for _dep in _dep_list :
_status, _msg = execute_command("configure", _dep, depsdict, \
hostname = "127.0.0.1", \
username = username, \
venv = options.venv, \
raise_exception = _raise_exception)
if _status :
_dep_missing += 1
_missing_dep.append(_dep)
cberr(_msg)
if operation == "install" :
_status, _msg = execute_command("install", _dep, depsdict, \
hostname = "127.0.0.1", \
username = username, \
venv = options.venv, \
raise_exception = _raise_exception)
if not _status :
_dep_missing -= 1
_missing_dep.remove(_dep)
cbinfo(_msg)
else :
cberr(_msg)
else :
cbinfo(_msg)
_status = _dep_missing
_fmsg += ','.join(_missing_dep)
except KeyError, e:
_status = 22
_fmsg = "Unable to find entry " + str(e) + " in dependencies dictionary. Check you dependencies configuration file(s)"
def build_repository_file_contents(depsdict, repo_name) :
'''
TBD
'''
_msg = "Configuring repository \"" + repo_name +"\"..."
cbinfo(_msg)
_file_contents = ""
if "local-url" in depsdict["repo_contents"][repo_name] :
if len(depsdict["repo_contents"][repo_name]["local-url"]) :
if not depsdict["repo_addr"] and \
depsdict["repo_contents"][repo_name]["local-url"].count("REPO_ADDR") :
_actual_url = depsdict["repo_contents"][repo_name]["original-url"]
else :
_actual_url = depsdict["repo_contents"][repo_name]["local-url"]
else :
_actual_url = depsdict["repo_contents"][repo_name]["original-url"]
if depsdict["repo_addr"] :
_actual_url = _actual_url.replace("REPO_ADDR", depsdict["repo_addr"])
_actual_url = _actual_url.replace("REPO_RELEASE", depsdict["cdistver"])
_actual_url = _actual_url.replace("REPO_MAJOR_RELEASE", depsdict["cdistmajorver"])
_actual_url = _actual_url.replace("REPO_ARCH", depsdict["carch"])
if not check_url(_actual_url, "ARCH", depsdict["carch"]) :
_tested_urls = _actual_url
_actual_url = depsdict["repo_contents"][repo_name]["original-url"]
if not check_url(_actual_url, "ARCH", depsdict["carch"]) :
if not _tested_urls.count(_actual_url) :
_tested_urls += ',' + _actual_url
_actual_url = False
if _actual_url :
_msg = "Valid URL found: " + _actual_url + "."
cbinfo(_msg)
else :
_msg = "No URLs available for repository \"" + repo_name
_msg += "\" (" + _tested_urls + ")." + " Will ignore this repository"
_msg += ", but this might cause installation errors due to a lacking on certain dependencies"
cbwarn(_msg)
return False
if depsdict["cdistkind"] == "ubuntu" :
for _dist in depsdict["repo_contents"][repo_name]["dists"].split(',') :
for _component in depsdict["repo_contents"][repo_name]["components"].split(',') :
_file_contents += "deb " + _actual_url + ' ' + _dist + ' ' + _component + "\n"
else :
_file_contents += "[" + repo_name + "]\n"
_file_contents += "name = " + repo_name + "\n"
_file_contents += "baseurl = " + _actual_url + "\n"
for _attr in [ "enabled", "skip_if_unavailable", "priority", "gpgcheck" ] :
_file_contents += _attr + " = " + depsdict["repo_contents"][repo_name][_attr] + "\n"
if depsdict["repo_contents"][repo_name]["gpgcheck"] == "0" :
True
else :
_file_contents += "gpgkey = " + depsdict["repo_contents"][repo_name]["gpgkey"] + "\n"
return _file_contents
def deps_file_parser(depsdict,
username,
options,
hostname,
process_manager=False):
'''
TBD
'''
_file_name_list = []
_file_name_list.append(options.defdir + "/PUBLIC_dependencies.txt")
_cleanup_repos = False
if len(options.wks) > 1:
_workloads_list = options.wks.split(',')
for _workload in _workloads_list:
_file_name_list.append(options.wksdir + '/' + _workload +
"/dependencies.txt")
_cleanup_repos = True
_file_name_list.append(options.defdir + "/IBM_dependencies.txt")
_file_name_list.append(options.defdir + "/SPEC_dependencies.txt")
if len(options.custom):
_file_name_list.append(options.cusdir + '/' + options.custom)
print '\n'
for _file in _file_name_list:
if os.access(_file, os.F_OK):
try:
_fd = open(_file, 'r')
_fc = _fd.readlines()
_fd.close()
_msg = "##### File \"" + _file + "\" opened and loaded...."
cbinfo(_msg)
for _line in _fc:
_line = _line.strip()
if _line.count("#", 0, 2):
_sstr = None
elif len(_line) < 3:
_sstr = None
elif _line.count(" = "):
_sstr = " = "
elif _line.count(" ="):
_sstr = " ="
elif _line.count("= "):
_sstr = "= "
elif _line.count("="):
_sstr = "="
else:
_sstr = None
if _sstr:
_key, _value = _line.split(_sstr)
_key = _key.strip()
depsdict[_key] = _value
except Exception, e:
_msg = "##### Error reading file \"" + _file + "\":" + str(e)
cberr(_msg)
exit(4)
else:
_msg = "##### File \"" + _file + "\" IGNORED...."
cbwarn(_msg)
def docker_file_parser(depsdict,
username,
options,
hostname,
process_manager=False):
'''
TBD
'''
_workloads_list = []
if options.role.count("orchestrator"):
_workloads_list = ["orchestrator", "orchprereqs"]
_role_list = [options.role, "orchprereqs"]
if len(options.wks) > 1:
_workloads_list = options.wks.split(',')
_role_list = [options.role]
print '\n'
for _role in _role_list:
for _path, _dirs, _files in os.walk(options.dfdir + '/' + _role):
for _fnam in _files:
_full_fnam = os.path.join(_path, _fnam)
if not _fnam.count("._processed_") and _fnam.count(
"Dockerfile-") and os.access(_full_fnam, os.F_OK):
_x_fnam = _fnam.replace("Dockerfile-", '').split('_')
_key_prefix = _x_fnam[0]
_f_workload = _x_fnam[2]
if len(_x_fnam) > 3:
_f_workload += '_' + _x_fnam[3]
if _f_workload in _workloads_list:
try:
_fd = open(_full_fnam, 'r')
_fc = _fd.readlines()
_fd.close()
_msg = "##### Parsing Dockerfile \"" + _full_fnam + "\"...."
cbinfo(_msg)
_current_key = None
for _line in _fc:
_line = _line.strip()
if _current_key:
if _line.count("#") and _line.count(
"-install-"):
depsdict[_current_key] = depsdict[
_current_key][0:-1]
if _current_key.count("centos-"):
depsdict[_current_key.replace(
"centos-", "rhel-"
)] = depsdict[_current_key]
depsdict[_current_key.replace(
"centos-", "fedora-"
)] = depsdict[_current_key]
_current_key = None
elif _line.count("#") and _line.count(
"RUN"):
True
else:
_line = _line.replace(
"RUN apt-get install -y",
"package_install")
_line = _line.replace(
"RUN yum install -y",
"package_install")
_line = _line.replace(
"RUN pip install --upgrade",
"sudo pip install --upgrade INDEXURL"
)
_line = _line.replace("RUN git", "git")
_line = _line.replace(
"; apt-get install",
"; sudo apt-get install")
_line = _line.replace(
"; apt-get update",
"; sudo apt-get update")
_line = _line.replace(
"; add-apt-repository",
"; sudo add-apt-repository")
_line = _line.replace(
"; dpkg", "; sudo dpkg")
_line = _line.replace(
"; yum install",
"; sudo yum install")
_line = _line.replace(
"; rpm", "; sudo rpm")
_line = _line.replace(
"; chown", "; sudo chown")
_line = _line.replace(
"; make install",
"; sudo make install")
_line = _line.replace(
"RUN mkdir -p /home/REPLACE_USERNAME/cbtool/3rd_party",
"mkdir -p 3RPARTYDIR")
_line = _line.replace("RUN ", "sudo ")
_line = _line.replace(
"ENV ", "export ")
_line = _line.replace(
"sudo cd ", "cd ")
_line = _line.replace(
"sudo export", "export ")
_line = _line.replace(
"sudo sudo ", "sudo ")
_line = _line.replace(
"sudo REPLACE_RSYNC",
"REPLACE_RSYNC")
_line = _line.replace(
"WORKDIR /home/REPLACE_USERNAME/cbtool/3rd_party",
"cd 3RPARTYDIR")
_line = _line.replace(
"# service_stop_disable",
"service_stop_disable")
_line = _line.replace("# echo", "echo")
_line = _line.replace(
"/home/root", "/root")
_line = _line.replace("____", ' ')
depsdict[_current_key] += _line + "; "
else:
if _line.count("#") and _line.count(
"-install-"):
_current_key = _key_prefix + '-' + _line.replace(
"#", '').strip()
depsdict[_current_key] = ''
except Exception, e:
_msg = "##### Error reading file \"" + _full_fnam + "\":" + str(
e)
cberr(_msg)
exit(4)
def dependency_checker_installer(hostname, depsdict, username, operation,
options):
'''
TBD
'''
try:
_status = 100
_dep_missing = -1
_fmsg = "An error has occurred, but no error message was captured"
if len(options.wks) > 1:
if options.wks.count("_ycsb"):
options.wks += ",ycsb"
if options.wks.count(",ycsb"):
options.wks += ",mongo_ycsb,cassandra_ycsb,redis_ycsb"
if options.wks.count(",acmeair"):
options.wks += ",mongo_acmeair"
deps_file_parser(depsdict, username, options, "127.0.0.1")
docker_file_parser(depsdict, username, options, "127.0.0.1")
preparation_file_parser(depsdict, username, options, "127.0.0.1")
if "Filestore_ip" not in depsdict:
depsdict["Filestore_ip"], depsdict["Filestore_port"], depsdict[
"Filestore_username"] = options.filestore.split('-')
depsdict["cdistkind"], depsdict["cdistver"], depsdict[
"cdistmajorver"], depsdict["cdistnam"], depsdict[
"carch"] = get_linux_distro()
depsdict["3rdpartydir"] = options.tpdir
depsdict["scriptsdir"] = options.wksdir
depsdict["credentialsdir"] = options.creddir
depsdict["username"] = username
if options.addr:
depsdict["repo-addr1"] = options.addr
depsdict["pip-addr1"] = options.addr
if depsdict["carch"] == "x86_64":
depsdict["carch1"] = "x86_64"
depsdict["carch2"] = "x86-64"
depsdict["carch3"] = "amd64"
elif depsdict["carch"] == "ppc64le":
depsdict["carch1"] = "ppc64le"
depsdict["carch2"] = "ppc64"
depsdict["carch3"] = "ppc64"
else:
depsdict["carch1"] = "aarch64"
depsdict["carch2"] = "aarch64"
depsdict["carch3"] = "aarch64"
_missing_dep = []
_dep_list = [0] * 5000
if str(options.addr) != "bypass":
select_url("repo", depsdict)
select_url("pip", depsdict)
_raise_exception = True
else:
depsdict["pip_addr"] = None
depsdict["repo_addr"] = None
_raise_exception = False
for _key in depsdict.keys():
if _key.count("-order"):
_dependency = _key.replace("-order", '')
_order = int(depsdict[_key]) * 20
_dep_list.insert(_order, _dependency)
_dep_list = [x for x in _dep_list if x != 0]
print '\n'
if options.role.count("workload"):
options.tag = "base," + options.role
_msg = "##### This node will be used to play a role in the Virtual Applications"
_msg += " (AIs) \"" + str(
options.wks) + "\". Only a subset of the depedencies"
_msg += " will be " + operation + "ed. This node cannot be used as an Orchestrator Node\n"
_msg += "\n"
cbinfo(_msg)
else:
options.tag = "base," + options.role + ',' + options.clouds
_msg = "##### This node will be prepared as an Orchestration Node."
_msg += " The full set of dependencies will be " + operation + "ed. "
_msg += "\n"
cbinfo(_msg)
options.tag = options.tag.split(',')
_selected_dep_list = []
for _dep in _dep_list:
for _tag in options.tag:
if _dep + "-tag" in depsdict:
_dep_tag_list = depsdict[_dep + "-tag"].split(',')
else:
_dep_tag_list = ["workload"]
if _tag in _dep_tag_list:
if _dep not in _selected_dep_list:
_selected_dep_list.append(_dep)
_dep_list = _selected_dep_list
_process_manager = ProcessManagement(hostname)
_status, _std_out, _y = _process_manager.run_os_command(
"sudo cat /proc/1/cgroup | grep -c docker", raise_exception=False)
if _status:
depsdict["indocker"] = False
else:
if str(_std_out.replace("\n", '')) == '0':
depsdict["indocker"] = False
else:
depsdict["indocker"] = True
_msg = "##### DETECTED OPERATING SYSTEM KIND: " + depsdict["cdistkind"]
cbinfo(_msg)
_msg = "##### DETECTED OPERATING SYSTEM VERSION: " + depsdict[
"cdistver"] + " (" + depsdict["cdistmajorver"] + ')'
cbinfo(_msg)
_msg = "##### DETECTED OPERATING SYSTEM NAME: " + depsdict["cdistnam"]
cbinfo(_msg)
_msg = "##### DETECTED ARCHITECTURE: " + depsdict["carch"]
cbinfo(_msg)
_msg = "##### DETECTED RUNNING INSIDE DOCKER: " + str(
depsdict["indocker"])
cbinfo(_msg)
print '\n'
if operation == "configure":
if "repo" in _dep_list:
_dep_list.remove("repo")
if depsdict["cdistkind"] == "AMI":
_msg = "This node runs the \"" + depsdict["cdistkind"] + "\" Linux "
_msg += "distribution. Will treat it as \"rhel\", but will disable"
_msg += " the repository manipulation."
cbinfo(_msg)
depsdict["cdistkind"] = "rhel"
if "repo" in _dep_list:
_dep_list.remove("repo")
if depsdict["carch"].count("ppc") and "mongdob" in _dep_list:
_msg = "##### The processors on this node have a \"Power\" architecture."
_msg += "Removing MongoDB and Chef (client) from the dependency list"
cbwarn(_msg)
_dep_list.remove("mongodb")
_dep_list.remove("chef-client")
if "java" in _dep_list and "oraclejava" in _dep_list:
_msg = "Since both \"java\" and \"oraclejava\" are listed as dependencies"
_msg += ", only \"oraclejava\" will be used"
cbinfo(_msg)
_dep_list.remove("java")
_dep_list.remove("java-home")
_fmsg = ""
_dep_missing = 0
for _dep in _dep_list:
_status, _msg = execute_command("configure", _dep, depsdict, \
hostname = "127.0.0.1", \
username = username, \
venv = options.venv, \
raise_exception = _raise_exception)
if _status:
_dep_missing += 1
_missing_dep.append(_dep)
cberr(_msg)
if operation == "install":
_status, _msg = execute_command("install", _dep, depsdict, \
hostname = "127.0.0.1", \
username = username, \
venv = options.venv, \
raise_exception = _raise_exception)
if not _status:
_dep_missing -= 1
_missing_dep.remove(_dep)
cbinfo(_msg)
else:
cberr(_msg)
else:
cbinfo(_msg)
_status = _dep_missing
_fmsg += ','.join(_missing_dep)
except KeyError, e:
_status = 22
_fmsg = "Unable to find entry " + str(
e
) + " in dependencies dictionary. Check you dependencies configuration file(s)"
if _status:
if operation == "install":
_msg += "There was an error while installing \"" + depkey + "\".: "
_msg += _result_stderr + "\n"
else:
_msg += " CORRECTIVE ACTION: " + inst_conf_msg(
depkey, depsdict)
if depkey == "sudo":
_msg = "Before proceeding further: " + inst_conf_msg(
"sudo", depsdict)
_msg += " *AS ROOT*"
cberr(_msg)
exit(20)
else:
cbinfo(_msg)
return _status, _msg
def compare_versions(depkey, depsdict, version_b):
'''
TBD
'''
try:
version_a = depsdict[depkey + "-ver"]
if version_a.lower() == "any":
_result = 0
version_a = "ANY"
version_b = "ANY"
else:
def deps_file_parser(depsdict, username, options, hostname, process_manager = False) :
'''
TBD
'''
_file_name_list = []
_file_name_list.append(options.defdir + "/PUBLIC_dependencies.txt")
_cleanup_repos = False
if len(options.wks) > 1 :
_workloads_list = options.wks.split(',')
for _workload in _workloads_list :
_file_name_list.append(options.wksdir + '/' + _workload + "/dependencies.txt")
_cleanup_repos = True
_file_name_list.append(options.defdir + "/IBM_dependencies.txt")
_file_name_list.append(options.defdir + "/SPEC_dependencies.txt")
if len(options.custom) :
_file_name_list.append(options.cusdir + '/' + options.custom)
print('\n')
for _file in _file_name_list :
if os.access(_file, os.F_OK) :
try:
_fd = open(_file, 'r')
_fc = _fd.readlines()
_fd.close()
_msg = "##### File \"" + _file + "\" opened and loaded...."
cbinfo(_msg)
for _line in _fc :
_line = _line.strip()
if _line.count("#",0,2) :
_sstr = None
elif len(_line) < 3 :
_sstr = None
elif _line.count(" = ") :
_sstr = " = "
elif _line.count(" =") :
_sstr = " ="
elif _line.count("= ") :
_sstr = "= "
elif _line.count("=") :
_sstr = "="
else :
_sstr = None
if _sstr :
_key, _value = _line.split(_sstr)
_key = _key.strip()
depsdict[_key] = _value
except Exception as e :
_msg = "##### Error reading file \"" + _file + "\":" + str(e)
cberr(_msg)
exit(4)
else :
_msg = "##### File \"" + _file + "\" IGNORED...."
cbwarn(_msg)
if not len(depsdict) :
_msg = "##### None of the files on the list \"" + str(_file_name_list)
_msg += "\" contained configuration statements"
cberr(_msg)
exit(9)
if _cleanup_repos :
if not process_manager :
process_manager = ProcessManagement(hostname)
process_manager.run_os_command("sudo rm -rf /tmp/repoupdated", False)
return True
def instance_preparation(hostname, depsdict, options) :
'''
TBD
'''
try :
_status = 100
_store = False
_process_manager = False
_cleanup = False
_fmsg = "An error has occurred, but no error message was captured"
if "stores" in depsdict :
for _store in depsdict["stores"] :
_process_manager = ProcessManagement(hostname)
_cmd = depsdict[_store + "_command_check"]
_msg = "Checking accesss from this instance to the CB"
_msg += " Orchestrator's " + _store.capitalize() + " with"
_msg += " \"" + _cmd + "\"..."
cbinfo(_msg)
_status, _x, _y = _process_manager.run_os_command(_cmd)
_msg = "This instance was able to access CB Orchestrator's " + _store.capitalize()
_msg += " with \"" + _cmd + "\" (OK)"
cbinfo(_msg)
if _store.lower() == "filestore" :
print('\n')
_msg = "rsync rsync://" + depsdict["Filestore_ip"] + ':' + depsdict["Filestore_port"] + '/' + depsdict["Filestore_username"] + "_cb"
print(_msg)
print('\n')
_msg = "--filestore " + depsdict["Filestore_ip"] + '-' + depsdict["Filestore_port"] + '-' + depsdict["Filestore_username"]
print(_msg)
print('\n')
if str(options.addr) != "bypass" :
_cmd = options.wksdir + "/common/cb_cleanup.sh"
_msg = "Running the instance cleanup script \"" + _cmd + "\"..."
cbinfo(_msg)
_process_manager.run_os_command(_cmd)
_cleanup = True
_status = 0
except Exception as e :
_status = 23
_fmsg = str(e)
finally :
if _status :
if _store and not _cleanup :
_msg = "ERROR while checking accesss from this instance to the CB"
_msg += " Orchestrator's " + _store.capitalize() + ": " + _fmsg
_msg += " (make sure that the " + depsdict[_store + "_protocol"]
_msg += " port " + depsdict[_store + "_port"]
_msg += " is open at IP address " + depsdict[_store + "_ip"]
if not _store and not _cleanup :
_msg = "ERROR while preparing to check access from this instance "
_msg += "to the CB Orchestrator's stores:" + _fmsg
if _store and _cleanup :
_msg = "ERROR while cleaning up instance."
cberr(_msg)
else :
_msg = ''
if _process_manager :
_msg += "Sucessfully checked this instance's ability to connect"
_msg += " to the Orchestrator's stores."
if _cleanup and str(options.addr).lower() != "bypass" :
_msg += "Sucessfully cleaned up this instance."
cbinfo(_msg)
return _status, _msg
def execute_command(operation, depkey, depsdict, hostname = "127.0.0.1", username = None, process_manager = None, venv = False, raise_exception = False):
'''
TBD
'''
try :
_status = 100
_msg = "Obtaining command to be executed...."
_status = 20000
_result_stdout = "NA"
_result_stderr = "NA"
if not process_manager :
process_manager = ProcessManagement(hostname)
_cmd = {}
_cmd["configure-keys"], _cmd["configure"] = get_cmdline(depkey, depsdict, "configure", process_manager, raise_exception)
_cmd["install-keys"], _cmd["install"] = get_cmdline(depkey, depsdict, "install", process_manager, raise_exception)
_order = depsdict[depkey + "-order"]
if depkey != "sudo" and operation == "configure" :
_msg = "(" + _order + ") Checking \"" + depkey + "\" version by executing the command \""
_msg += _cmd[operation] + "\" (" + _cmd[operation + "-keys"] + ")..."
elif depkey == "sudo" and operation == "configure" :
_msg = "(" + _order + ") Checking passwordless sudo for the user \"" + username + "\" "
_msg += "by executing the command \"" + _cmd[operation] + "\" (" + _cmd[operation + "-keys"] + ")..."
else :
if venv :
_cmd["install"] = _cmd["install"].replace("sudo pip", "pip")
_msg = "(" + _order + ") Installing \"" + depkey + "\" by executing the command \""
_msg += _cmd[operation] + "\" (" + _cmd[operation + "-keys"] + ")..."
cbinfo(_msg)
_msg = "(" + _order + ") RESULT for command \"" + _cmd[operation] + "\" : "
if depkey == "repo" and operation == "install" :
build_repository_files(depsdict)
_status, _result_stdout, _result_stderr = process_manager.run_os_command(_cmd[operation], False, True, False, True, None, False, 22, False)
if not _status :
if operation == "install" :
_msg += "DONE OK.\n"
else :
_msg += compare_versions(depkey, depsdict, _result_stdout.strip())
else :
_msg += "NOT OK (exit code " + str(_status) + "). "
if _msg.count("NOT OK") :
_status = 701
except ProcessManagement.ProcessManagementException, obj :
_status = str(obj.status)
_result_stderr = str(obj.msg)
_msg += "NOT OK (PMgr Exception, exit code " + str(_status) + "). "
def build_repository_file_contents(depsdict, repo_name):
'''
TBD
'''
_msg = "Configuring repository \"" + repo_name + "\"..."
cbinfo(_msg)
_file_contents = ""
if "local-url" in depsdict["repo_contents"][repo_name]:
if len(depsdict["repo_contents"][repo_name]["local-url"]):
if not depsdict["repo_addr"] and \
depsdict["repo_contents"][repo_name]["local-url"].count("REPO_ADDR") :
_actual_url = depsdict["repo_contents"][repo_name][
"original-url"]
else:
_actual_url = depsdict["repo_contents"][repo_name]["local-url"]
else:
_actual_url = depsdict["repo_contents"][repo_name]["original-url"]
if depsdict["repo_addr"]:
_actual_url = _actual_url.replace("REPO_ADDR", depsdict["repo_addr"])
_actual_url = _actual_url.replace("REPO_RELEASE", depsdict["cdistver"])
_actual_url = _actual_url.replace("REPO_MAJOR_RELEASE",
depsdict["cdistmajorver"])
_actual_url = _actual_url.replace("REPO_ARCH", depsdict["carch"])
if not check_url(_actual_url, "ARCH", depsdict["carch"]):
_tested_urls = _actual_url
_actual_url = depsdict["repo_contents"][repo_name]["original-url"]
if not check_url(_actual_url, "ARCH", depsdict["carch"]):
if not _tested_urls.count(_actual_url):
_tested_urls += ',' + _actual_url
_actual_url = False
if _actual_url:
_msg = "Valid URL found: " + _actual_url + "."
cbinfo(_msg)
else:
_msg = "No URLs available for repository \"" + repo_name
_msg += "\" (" + _tested_urls + ")." + " Will ignore this repository"
_msg += ", but this might cause installation errors due to a lacking on certain dependencies"
cbwarn(_msg)
return False
if depsdict["cdistkind"] == "ubuntu":
for _dist in depsdict["repo_contents"][repo_name]["dists"].split(','):
for _component in depsdict["repo_contents"][repo_name][
"components"].split(','):
_file_contents += "deb " + _actual_url + ' ' + _dist + ' ' + _component + "\n"
else:
_file_contents += "[" + repo_name + "]\n"
_file_contents += "name = " + repo_name + "\n"
_file_contents += "baseurl = " + _actual_url + "\n"
for _attr in [
"enabled", "skip_if_unavailable", "priority", "gpgcheck"
]:
_file_contents += _attr + " = " + depsdict["repo_contents"][
repo_name][_attr] + "\n"
if depsdict["repo_contents"][repo_name]["gpgcheck"] == "0":
True
else:
_file_contents += "gpgkey = " + depsdict["repo_contents"][
repo_name]["gpgkey"] + "\n"
return _file_contents
def vmresize_actual(self, obj_attr_list) :
'''
TBD
'''
cbdebug("VM " + obj_attr_list["name"] + " resize request sent.", True)
_vg = ValueGeneration(self.pid)
tag = obj_attr_list["cloud_uuid"]
desc = obj_attr_list["resource_description"]
hypervisor_ip = obj_attr_list["vmc_cloud_ip"]
if not self.is_vm_running(obj_attr_list) :
return 0, "VM is not running"
cbdebug("Getting resource state for Guest " + tag)
_status, _fmsg, _guest_info = self.plmconn.get_domain_full_info(tag, hypervisor_ip)
try :
if "cpu_nr" in desc :
_msg = "Set the number of CPUs for Guest " + tag
_msg = "(" + obj_attr_list["cloud_ip"] + ")."
cbdebug(_msg)
_hpg_cnt = HotplugMgdConn(self.pid, obj_attr_list["cloud_ip"], "root", obj_attr_list["identity"])
_active_cpus = 0
_cpu_list = _hpg_cnt.get_cpus_state()
for _cpu_number, _cpu_state in enumerate(_cpu_list) :
if _cpu_state == '1' :
_active_cpus += 1
_hpg_cnt.set_active_cpus(desc["cpu_nr"])
_msg = "CPU number for Guest " + tag
_msg += " (" + obj_attr_list["cloud_ip"] + ") set to " + str(desc["cpu_nr"])
_msg += " from " + str(_active_cpus) + '.'
_xmsg = _msg
cbdebug(_msg, True)
del desc["cpu_nr"]
if "cpu_sl" in desc :
cbdebug("Setting CPU Soft Limit for Guest " + tag)
_cpu_sl = _vg.value_suffix(desc["cpu_sl"], False)
self.plmconn.set_domain_cpu(tag, "cpu_shares", str(_cpu_sl), hypervisor_ip)
_msg = "CPU Soft Limit for Guest \"" + obj_attr_list["cloud_uuid"]
_msg += "\" successfully set to " + str(_cpu_sl) + " from "
_msg += str(_guest_info["vcpus_soft_limit"]) + '.'
cbdebug(_msg, True)
del desc["cpu_sl"]
if "cpu_hl" in desc :
_msg = "Setting CPU Hard Limit for Guest \"" + obj_attr_list["cloud_uuid"]
_msg += "\"."
cbdebug(_msg)
_cpu_hl = int(float(desc["cpu_hl"]) * \
float(_guest_info["vcpus_period"]))
self.plmconn.set_domain_cpu(tag, "vcpu_quota", str(_cpu_hl), hypervisor_ip)
_msg = "CPU Hard Limit for Guest " + tag
_msg += " successfully set to " + str(desc["cpu_hl"])
_msg += " from " + str(_guest_info["vcpus_hard_limit"]) + '.'
cbdebug(_msg, True)
del desc["cpu_hl"]
if "mem_sl" in desc :
cbdebug("Setting MEMORY Soft Limit for Guest " + tag)
_mem_sl = _vg.value_suffix(desc["mem_sl"], True)
cbdebug("Resource Control Not implemented", True)
del desc["mem_sl"]
if "mem_hl" in desc :
cbinfo("Setting MEMORY Hard Limit for Guest " + tag)
_mem_hl = _vg.value_suffix(desc["mem_hl"], True)
self.plmconn.set_domain_memory(tag, "current_memory", str(_mem_hl * 1024), hypervisor_ip)
_msg = "MEMORY Hard Limit (virtio balloon) for Guest " + tag
_msg += " successfully set to " + str(_mem_hl)
_msg += " KB from " + str(_guest_info["current_memory"]) + "KB ."
cbdebug(_msg, True)
del desc["mem_hl"]
except ValueGeneration.ValueGenerationException, obj :
raise CldOpsException("resize failure: " + obj.msg, obj.status)
def execute_command(operation,
depkey,
depsdict,
hostname="127.0.0.1",
username=None,
process_manager=None,
venv=False,
raise_exception=False):
'''
TBD
'''
try:
_status = 100
_msg = "Obtaining command to be executed...."
_status = 20000
_result_stdout = "NA"
_result_stderr = "NA"
if not process_manager:
process_manager = ProcessManagement(hostname)
_cmd = {}
_cmd["configure-keys"], _cmd["configure"] = get_cmdline(
depkey, depsdict, "configure", process_manager, raise_exception)
_cmd["install-keys"], _cmd["install"] = get_cmdline(
depkey, depsdict, "install", process_manager, raise_exception)
_order = depsdict[depkey + "-order"]
if depkey != "sudo" and operation == "configure":
_msg = "(" + _order + ") Checking \"" + depkey + "\" version by executing the command \""
_msg += _cmd[operation] + "\" (" + _cmd[operation +
"-keys"] + ")..."
elif depkey == "sudo" and operation == "configure":
_msg = "(" + _order + ") Checking passwordless sudo for the user \"" + username + "\" "
_msg += "by executing the command \"" + _cmd[
operation] + "\" (" + _cmd[operation + "-keys"] + ")..."
else:
if venv:
_cmd["install"] = _cmd["install"].replace("sudo pip", "pip")
_msg = "(" + _order + ") Installing \"" + depkey + "\" by executing the command \""
_msg += _cmd[operation] + "\" (" + _cmd[operation +
"-keys"] + ")..."
cbinfo(_msg)
_msg = "(" + _order + ") RESULT for command \"" + _cmd[
operation] + "\" : "
if depkey == "repo" and operation == "install":
build_repository_files(depsdict)
_status, _result_stdout, _result_stderr = process_manager.run_os_command(
_cmd[operation], False, True, False, True, None, False, 22, False)
if not _status:
if operation == "install":
_msg += "DONE OK.\n"
else:
_msg += compare_versions(depkey, depsdict,
_result_stdout.strip())
else:
_msg += "NOT OK (exit code " + str(_status) + "). "
if _msg.count("NOT OK"):
_status = 701
except ProcessManagement.ProcessManagementException, obj:
_status = str(obj.status)
_result_stderr = str(obj.msg)
_msg += "NOT OK (PMgr Exception, exit code " + str(_status) + "). "
def vmresize_actual(self, obj_attr_list):
'''
TBD
'''
cbdebug("VM " + obj_attr_list["name"] + " resize request sent.", True)
_vg = ValueGeneration(self.pid)
tag = obj_attr_list["cloud_vm_uuid"]
desc = obj_attr_list["resource_description"]
hypervisor_ip = obj_attr_list["host_cloud_ip"]
if not self.is_vm_running(obj_attr_list):
return 0, "VM is not running"
cbdebug("Getting resource state for Guest " + tag)
_status, _fmsg, _guest_info = self.ftcconn.get_domain_full_info(
tag, hypervisor_ip)
try:
if "cpu_nr" in desc:
_msg = "Set the number of CPUs for Guest " + tag
_msg = "(" + obj_attr_list["cloud_ip"] + ")."
cbdebug(_msg)
_hpg_cnt = HotplugMgdConn(self.pid, obj_attr_list["cloud_ip"],
"root", obj_attr_list["identity"])
_active_cpus = 0
_cpu_list = _hpg_cnt.get_cpus_state()
for _cpu_number, _cpu_state in enumerate(_cpu_list):
if _cpu_state == '1':
_active_cpus += 1
_hpg_cnt.set_active_cpus(desc["cpu_nr"])
_msg = "CPU number for Guest " + tag
_msg += " (" + obj_attr_list["cloud_ip"] + ") set to " + str(
desc["cpu_nr"])
_msg += " from " + str(_active_cpus) + '.'
_xmsg = _msg
cbdebug(_msg, True)
del desc["cpu_nr"]
if "cpu_sl" in desc:
cbdebug("Setting CPU Soft Limit for Guest " + tag)
_cpu_sl = _vg.value_suffix(desc["cpu_sl"], False)
self.ftcconn.set_domain_cpu(tag, "cpu_shares", str(_cpu_sl),
hypervisor_ip)
_msg = "CPU Soft Limit for Guest \"" + obj_attr_list[
"cloud_vm_uuid"]
_msg += "\" successfully set to " + str(_cpu_sl) + " from "
_msg += str(_guest_info["vcpus_soft_limit"]) + '.'
cbdebug(_msg, True)
del desc["cpu_sl"]
if "cpu_hl" in desc:
_msg = "Setting CPU Hard Limit for Guest \"" + obj_attr_list[
"cloud_vm_uuid"]
_msg += "\"."
cbdebug(_msg)
_cpu_hl = int(float(desc["cpu_hl"]) * \
float(_guest_info["vcpus_period"]))
self.ftcconn.set_domain_cpu(tag, "vcpu_quota", str(_cpu_hl),
hypervisor_ip)
_msg = "CPU Hard Limit for Guest " + tag
_msg += " successfully set to " + str(desc["cpu_hl"])
_msg += " from " + str(_guest_info["vcpus_hard_limit"]) + '.'
cbdebug(_msg, True)
del desc["cpu_hl"]
if "mem_sl" in desc:
cbdebug("Setting MEMORY Soft Limit for Guest " + tag)
_mem_sl = _vg.value_suffix(desc["mem_sl"], True)
cbdebug("Resource Control Not implemented", True)
del desc["mem_sl"]
if "mem_hl" in desc:
cbinfo("Setting MEMORY Hard Limit for Guest " + tag)
_mem_hl = _vg.value_suffix(desc["mem_hl"], True)
self.ftcconn.set_domain_memory(tag, "current_memory",
str(_mem_hl * 1024),
hypervisor_ip)
_msg = "MEMORY Hard Limit (virtio balloon) for Guest " + tag
_msg += " successfully set to " + str(_mem_hl)
_msg += " KB from " + str(
_guest_info["current_memory"]) + "KB ."
cbdebug(_msg, True)
del desc["mem_hl"]
except ValueGeneration.ValueGenerationException, obj:
raise CldOpsException("resize failure: " + obj.msg, obj.status)
finally :
if _status :
if operation == "install" :
_msg += "There was an error while installing \"" + depkey + "\".: "
_msg += _result_stderr + "\n"
else :
_msg += " CORRECTIVE ACTION: " + inst_conf_msg(depkey, depsdict)
if depkey == "sudo" :
_msg = "Before proceeding further: " + inst_conf_msg("sudo", depsdict)
_msg += " *AS ROOT*"
cberr(_msg)
exit(20)
else :
cbinfo(_msg)
return _status, _msg
def compare_versions(depkey, depsdict, version_b) :
'''
TBD
'''
try :
version_a = depsdict[depkey + "-ver"]
if version_a.lower() == "any" :
_result = 0
version_a = "ANY"
version_b = "ANY"
else :
_non_decimal = re.compile(r'[^\d.]+')
