def qsub(self, name, host, script, template=None, kind="dict"):
"""
Executes the qsub command on a given host.
NOTE this method may not yet be fully implemented
:param name: name of the script
:param host: host on which the script is to be run
:param script: The name of the script
:param template: The script is wrapped into a template
:param kind: The return is passed as dict, yaml, xml
:return:
"""
self.jobid_incr()
jobscript = self.create_script(name, script, template)
# copy the script to the remote host
self._write_to_file(jobscript, name)
# copy script to remote host
remote_path = self.data.get("cloudmesh", "pbs", host, "scripts")
print(remote_path)
xmkdir(host, remote_path)
manager_host = self.manager(host)
# call qsub on the remot host
r = Shell.scp(name, manager_host + ":" + remote_path)
jobid = Shell.ssh(manager_host,
"qsub {0}/{1}".format(remote_path, name)).rstrip()
return self.jobstatus(host, jobid, kind=kind)
def run(self):
banner("Setup the cloudmesh management yaml files ")
yamlfiles = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml']
dir_path = path_expand("~/.cloudmesh/{0}".format("accounts"))
if not os.path.exists(dir_path):
Shell.mkdir(dir_path)
for yamlfile in yamlfiles:
filename = path_expand("~/.cloudmesh/{0}/{1}".format(
"accounts", yamlfile))
if os.path.isfile(filename):
Console.error(
"File {0} already exists. If you like to reinstall it, please remove the file"
.format(yamlfile))
else:
Console.info("Copying file: {0} -> {1} ".format(
path_expand("etc/{0}/{1}".format("accounts", yamlfile)),
filename))
shutil.copy(
"etc/{0}/{1}".format("accounts", yamlfile),
path_expand("~/.cloudmesh/{0}/{1}".format(
"accounts", yamlfile)))
def run(self):
banner("Reset the cloudmesh management yaml files ")
yaml_files = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml']
dir_path = path_expand("~/.cloudmesh/{0}".format("accounts"))
if not os.path.exists(dir_path):
Shell.mkdir(dir_path)
for yaml_file in yaml_files:
filename = path_expand("~/.cloudmesh/{0}/{1}".format(
"accounts", yaml_file))
if os.path.isfile(filename):
Console.info("Removing file: {0}".format(filename))
Shell.rm(filename)
Console.info("Copying file: {0} -> {1} ".format(
path_expand("etc/{0}/{1}".format("accounts", yaml_file)),
filename))
shutil.copy(
"etc/{0}/{1}".format("accounts", yaml_file),
path_expand("~/.cloudmesh/{0}/{1}".format(
"accounts", yaml_file)))
else:
Console.info("Copying file: {0} -> {1} ".format(
path_expand("etc/{0}/{1}".format("accounts", yaml_file)),
filename))
shutil.copy(
"etc/{0}/{1}".format("accounts", yaml_file),
path_expand("~/.cloudmesh/{0}/{1}".format(
"accounts", yaml_file)))
def execute(self, name, command):
"""executes the command on the named host"""
if name in ["localhost"]:
r = '\n'.join(Shell.sh("-c", command).split()[-1:])
else:
r = '\n'.join(Shell.ssh(name, command).split()[-1:])
return r
def execute(self, name, command):
"""executes the command on the named host"""
if name in ["localhost"]:
r = '\n'.join(Shell.sh("-c", command).split()[-1:])
else:
r = '\n'.join(Shell.ssh(name, command).split()[-1:])
return r
def qsub(self, name, host, script, template=None, kind="dict"):
"""
Executes the qsub command on a given host.
NOTE this method may not yet be fully implemented
:param name: name of the script
:param host: host on which the script is to be run
:param script: The name of the script
:param template: The script is wrapped into a template
:param kind: The return is passed as dict, yaml, xml
:return:
"""
self.jobid_incr()
jobscript = self.create_script(name, script, template)
# copy the script to the remote host
self._write_to_file(jobscript, name)
# copy script to remote host
remote_path = self.data.get("cloudmesh", "pbs", host, "scripts")
print(remote_path)
xmkdir(host, remote_path)
manager_host = self.manager(host)
# call qsub on the remot host
r = Shell.scp(name, manager_host + ":" + remote_path)
jobid = Shell.ssh(manager_host, "qsub {0}/{1}".format(remote_path, name)).rstrip()
return self.jobstatus(host, jobid, kind=kind)
def open(self, filename=None):
if filename is not None:
self.filename = filename
else:
self.filename = path_expand(self.pbs.database_filename())
path = os.path.dirname(self.filename)
Shell.mkdir(path)
self.load()
def create_cloudmesh_yaml(filename):
if not os.path.exists(filename):
path = os.path.dirname(filename)
if not os.path.isdir(path):
Shell.mkdir(path)
etc_path = os.path.dirname(cloudmesh_client.__file__)
etc_file = os.path.join(etc_path, "etc", "cloudmesh.yaml")
to_dir = path_expand("~/.cloudmesh")
shutil.copy(etc_file, to_dir)
Console.ok("~/.cloudmesh/cloudmesh.yaml created")
def create_config(cls, username):
dir_path = path_expand("~/.cloudmesh/{0}".format("accounts"))
if not os.path.exists(dir_path):
Shell.mkdir(dir_path)
filename = path_expand("~/.cloudmesh/{0}/{1}".format("accounts", ".config"))
data = dict(user=username)
with open(filename, 'w') as outfile:
outfile.write(yaml.dump(data, default_flow_style=True))
def delete(cls, cluster, job, group=None):
"""
This method is used to terminate a job with the specified or a group of jobs
job_id or job_name in a given cluster
:param group:
:param cluster: the cluster like comet
:param job: the job id or name
:return: success message or error
"""
try:
if group is not None:
# get the job ids from the db
cm = CloudmeshDatabase()
arguments = {'cluster': cluster,
'group': group}
db_jobs = cm.find('batchjob',
**arguments)
list1 = []
for i in db_jobs:
list1.append(db_jobs[i]['job_id'])
# read active jobs
active_jobs = json.loads(cls.queue(cluster))
list2 = []
for i in active_jobs:
list2.append(active_jobs[i]['jobid'])
# find intersection
res = set(list1).intersection(set(list2))
if res is not None:
for j in res:
cmd = 'scancel {}'.format(str(j))
Shell.ssh(cluster, cmd)
print("Deleted {}".format(j))
return "All jobs for group {} killed successfully".format(group)
else:
args = 'scancel '
if job.isdigit():
args += job
else:
args += "-n {}".format(job)
Shell.ssh(cluster, args)
return "Job {} killed successfully".format(job)
except Exception as ex:
print("in exceptio")
print(ex)
return ex
def run(self):
banner("Setup the cmd3.yaml file")
cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml")
if os.path.isfile(cmd3_yaml):
print ("ERROR: the file {0} already exists".format(cmd3_yaml))
print
print ("If you like to reinstall it, please remove the file")
else:
print ("Copy file: {0} -> {1} ".format(path_expand("etc/cmd3.yaml"), cmd3_yaml))
Shell.mkdir("~/.cloudmesh")
shutil.copy("etc/cmd3.yaml", path_expand("~/.cloudmesh/cmd3.yaml"))
def run(self):
banner("Setup the cmd3.yaml file")
cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml")
if os.path.isfile(cmd3_yaml):
print ("ERROR: the file {0} already exists".format(cmd3_yaml))
print()
print ("If you like to reinstall it, please remove the file")
else:
print ("Copy file: {0} -> {1} ".format(path_expand("etc/cmd3.yaml"), cmd3_yaml))
Shell.mkdir("~/.cloudmesh")
shutil.copy("etc/cmd3.yaml", path_expand("~/.cloudmesh/cmd3.yaml"))
def start(self):
"""
Starts the cloudmesh_job process in the background.
"""
for key in ['dbpath', 'logpath']:
path = os.dirname(self.config[key])
Shell.mkdir(path)
r = Shell.sh("mongod", "--fork", "--logpath", self.config['logpath'],
"--prot", self.config['port'], '--dbpath',
self.config['dbpath'])
print(r)
# TODO
# get the id from r
self.config['id'] = None # put here the real id
def start(self):
try:
mongod = Shell.which("mongod")
command = [
mongod,
"--dbpath", str(self.db_path),
"--port", str(self.port),
"--fork",
"--logpath", str(self.log_file),
"--bind_ip", "127.0.0.1"
]
print(" ".join(command))
#a = subprocess.call(command)
os.system(" ".join(command))
os.system ("ps aux | fgrep mongod |fgrep -v fgrep")
Console.ok("MongoDB has been deployed")
self.info()
except Exception, e:
Console.error("we had a problem starting the mongo daemon")
print(e)
Console.error("MongoDB has stopped")
# TODO remove the exit in final code, for debugging only
sys.exit()
def _grep(self, search, platform):
if not search:
search = "'OS_PASSWORD': '******'"
cmd = "egrep -ri \"{0}\" * | cut -d\":\" -f1 > a.tmp".format(search)
print("[{0}]:{1}".format(platform, cmd))
os.system(cmd)
res = Shell.cat("a.tmp")
if res:
print ('[{0}]: [ERROR] PASSWORD(OR SECRET KEY) DETECTED, SEE FILES '
'BELOW'.format(platform))
print ("")
print (res)
else:
print ("[{0}]: NO PASSWORD DETECTED".format(platform))
Shell.rm("a.tmp")
print ("")
def stat(self, email):
"""
returns a statistic of a git author with the given e_mail.
:param email: name of the author
:rtype: a dict with the statistics
"""
result = Shell.git("log", "--all", "--stat",
'--author={0}'.format(email)).split("\n")
sums = [0, 0, 0]
for line in result:
if " files changed" in line:
line = line.strip()
line = line.replace(" insertions(+)", "")
line = line.replace(" insertion(+)", "")
line = line.replace(" deletion(-)", "")
line = line.replace(" deletions(-)", "")
line = line.replace(" files changed", "")
line = line.split(",")
data = [int(i) for i in line]
for index in range(0, len(data)):
sums[index] += data[index]
return {
"email": email,
"fileschanged": sums[0],
"inserted": sums[1],
"deleted": sums[2],
"lineschanged": sums[1] + sums[2]
}
def get_authors_by_date(header=False):
"""lists the authors of a git repository sorted by date.
Example:
0001 (2015-02-25): Gregor von Laszewski ([email protected])
0002 (2015-04-14): Fugang Wang ([email protected])
:rtype: str
"""
# modified from https://github.com/jgehrcke/git-authors
result = ""
# if header:
# result = result + "Authors\n"
# result = result + "=======\n\n"
r = Shell.git("log", "--encoding=utf-8", "--full-history", "--reverse",
"--format=format:%at;%an;%ae").split("\n")
seen = set()
for line in r:
timestamp, name, email = line.strip().split(";")
if name not in seen:
seen.add(name)
day = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp)))
result = result + "* {:04d} ({:}): {:} ({:})\n".format(
len(seen), day, name, email)
return result
def emails(self, output=None):
"""
returns the emails of the authors either as a text or as a dict. The
format is specified as an argument.
:param output: if "dict" is specified a dict will be returned
:rtype: dict or array of e-mails dependent on output
"""
format_string = "'%aN' <%cE>"
if output == 'dict':
format_string = "%aN\t%cE"
result = sorted(
set(
Shell.git("log", "--all",
"--format=" + format_string).split("\n")))
if output is None:
return result
elif output == "dict":
emails = {}
for l in result:
(name, email) = l.split("\t")
emails[name] = email
return emails
def stat(self, email):
"""
returns a statistic of a git author with the given e_mail.
:param email: name of the author
:rtype: a dict with the statistics
"""
result = Shell.git("log", "--all", "--stat", '--author={0}'.format(email)).split("\n")
sums = [0, 0, 0]
for line in result:
if " files changed" in line:
line = line.strip()
line = line.replace(" insertions(+)", "")
line = line.replace(" insertion(+)", "")
line = line.replace(" deletion(-)", "")
line = line.replace(" deletions(-)", "")
line = line.replace(" files changed", "")
line = line.split(",")
data = [int(i) for i in line]
for index in range(0, len(data)):
sums[index] += data[index]
return {"email": email,
"fileschanged": sums[0],
"inserted": sums[1],
"deleted": sums[2],
"lineschanged": sums[1] + sums[2]}
def get_authors_by_date(header=False):
"""lists the authors of a git repository sorted by date.
Example:
0001 (2015-02-25): Gregor von Laszewski ([email protected])
0002 (2015-04-14): Fugang Wang ([email protected])
:rtype: str
"""
# modified from https://github.com/jgehrcke/git-authors
result = ""
# if header:
# result = result + "Authors\n"
# result = result + "=======\n\n"
r = Shell.git("log",
"--encoding=utf-8",
"--full-history",
"--reverse",
"--format=format:%at;%an;%ae").split("\n")
seen = set()
for line in r:
timestamp, name, email = line.strip().split(";")
if name not in seen:
seen.add(name)
day = time.strftime("%Y-%m-%d", time.gmtime(float(timestamp)))
result = result + "* {:04d} ({:}): {:} ({:})\n".format(len(seen), day, name, email)
return result
def list_quotas(cls, cloud, format):
Quota.set_os_environment(cloud)
result = Shell.execute("nova", "quota-show")
d = Quota.convert_to_dict(result)
return dict_printer(d, order=['Quota',
'Limit'],
output=format)
def start(self):
"""
Starts the cloudmesh_job process in the background.
"""
for key in ['dbpath', 'logpath']:
path = os.dirname(self.config[key])
Shell.mkdir(path)
r = Shell.sh("mongod", "--fork",
"--logpath", self.config['logpath'],
"--prot", self.config['port'],
'--dbpath', self.config['dbpath'])
print (r)
# TODO
# get the id from r
self.config['id'] = None # put here the real id
def list(cls, cloud, start=None, end=None, tenant=None, format="table"):
# set the environment variables
set_os_environ(cloud)
try:
# execute the command
args = ["usage"]
if start is not None:
args.extend(["--start", start])
if end is not None:
args.extend(["--end", end])
if tenant is not None:
args.extend(["--tenant", tenant])
result = Shell.execute("nova", args)
result = Nova.remove_subjectAltName_warning(result)
lines = result.splitlines()
dates = lines[0]
# TODO: as stated below, nova returns additional lines,
# on my pc, SecurityWarning is returned, so filtering..
for l in lines[1:]:
if l.__contains__("SecurityWarning"):
lines.remove(l)
table = '\n'.join(lines[1:])
dates = dates.replace("Usage from ", "").replace("to", "").replace(" +", " ")[:-1].split()
#
# TODO: for some reason the nova command has returned not the
# first + char, so we could not ignore the line we may set - as
# additional comment char, but that did not work
#
d = TableParser.convert(table, comment_chars="+#")
# d["0"]["start"] = "start"
# d["0"]["end"] = "end"
d["0"]["start"] = dates[0]
d["0"]["end"] = dates[1]
# del d['0']
return dict_printer(d,
order=["start",
"end",
"servers",
"cpu hours",
"ram mb-hours",
"disk gb-hours"],
output=format)
except Exception, e:
return e
def image_list(self):
images = {}
lines = Shell.vagrant("box", "list")
for line in lines:
(name, kind) = line.split("(")
name = name.strip()
kind = kind.split(")")[0].strip()
images[name] = {"name": name, "kind": kind}
return images
def run(self):
banner("Setup the cloudmesh management yaml files ")
yamlfiles = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml']
dir_path = path_expand("~/.cloudmesh/{0}".format("accounts"))
if not os.path.exists(dir_path):
Shell.mkdir(dir_path)
for yamlfile in yamlfiles:
filename = path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yamlfile))
if os.path.isfile(filename):
Console.error("File {0} already exists. If you like to reinstall it, please remove the file".format(yamlfile))
else:
Console.info("Copying file: {0} -> {1} ".format(path_expand("etc/{0}/{1}".format("accounts", yamlfile)), filename))
shutil.copy("etc/{0}/{1}".format("accounts", yamlfile), path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yamlfile)))
def image_list(self):
images = {}
lines = Shell.vagrant("box", "list")
for line in lines:
(name, kind) = line.split("(")
name = name.strip()
kind = kind.split(")")[0].strip()
images[name] = {"name": name, "kind": kind}
return images
def deploy(self):
"""
creates the directories if they do not exist
"""
try:
r = Shell.mkdir(self.db_path)
except Exception, e:
Console.error("Problem creating the database directory {:}".format(self.db_path))
print(e)
def stop(self):
"""
Stops the cloudmesh_job process
"""
id = self.config['id']
# TODO use pythonic way to kill
# p = psutil.Process(pid)
# p.terminate() #or p.kill()
r = Shell.kill('-9', id)
pass
def delete(cls, cluster, job):
"""
This method is used to terminate a job with the specified
job_id or job_name in a given cluster
:param cluster: the cluster like comet
:param job: the job id or name
:return: success message or error
"""
try:
args = 'scancel '
if job.isdigit():
args += job
else:
args += "-n {}".format(job)
Shell.ssh(cluster, args)
return "Job {} killed successfully".format(job)
except Exception as ex:
return ex
def stop(self):
"""
Stops the cloudmesh_job process
"""
id = self.config['id']
# TODO use pythonic way to kill
# p = psutil.Process(pid)
# p.terminate() #or p.kill()
r = Shell.kill('-9', id)
pass
def run(self):
banner("Reset the cloudmesh management yaml files ")
yaml_files = ['cloudmesh_user.yaml', 'cloudmesh_project.yaml']
dir_path = path_expand("~/.cloudmesh/{0}".format("accounts"))
if not os.path.exists(dir_path):
Shell.mkdir(dir_path)
for yaml_file in yaml_files:
filename = path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yaml_file))
if os.path.isfile(filename):
Console.info("Removing file: {0}".format(filename))
Shell.rm(filename)
Console.info("Copying file: {0} -> {1} ".format(path_expand("etc/{0}/{1}".format("accounts", yaml_file)), filename))
shutil.copy("etc/{0}/{1}".format("accounts", yaml_file), path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yaml_file)))
else:
Console.info("Copying file: {0} -> {1} ".format(path_expand("etc/{0}/{1}".format("accounts", yaml_file)), filename))
shutil.copy("etc/{0}/{1}".format("accounts", yaml_file), path_expand("~/.cloudmesh/{0}/{1}".format("accounts", yaml_file)))
def create_cmd3_yaml_file(force=False, verbose=True):
def print_error(kind, path):
if verbose:
Console.error("the {0} {1} already exists".format(kind, path))
Console.msg("")
Console.msg("If you like to reinstall it, "
"please remove the file first")
Console.msg("")
if verbose:
banner("create cmd3.yaml")
cmd3_yaml = path_expand("~/.cloudmesh/cmd3.yaml")
if force or not os.path.isfile(cmd3_yaml):
Shell.mkdir(path_expand("~/.cloudmesh"))
import cmd3
content = pkg_resources.resource_string(cmd3.__name__, "etc/cmd3.yaml")
Console.ok("Generate yaml file")
with open(cmd3_yaml, "w") as cmd3_file:
cmd3_file.write(content)
else:
print_error('file', cmd3_yaml)
if verbose:
banner("create cmd3_template")
# # # copy tree
filename = '~/.cloudmesh/etc/cmd3_template'
if os.path.isdir(path_expand(filename)):
print_error('directory', filename)
else:
import glob
import shutil
import cmd3.etc.cmd3_template
f1 = cmd3.etc.cmd3_template.__file__
cmd3_etc_path = os.path.dirname(f1)
pattern = os.path.join(cmd3_etc_path, '*')
for src in glob.glob(pattern):
if os.path.isfile(src): continue
shutil.copytree(src, path_expand(filename))
def tag(self):
v = self.find()
banner("v")
print v
#v = ".".join(self.version)
os.system("python shell_plugins.py install")
if self.git_commit_needed():
banner("git commit")
command = "git commit -m 'version {:}' {:}".format(v,self.filename)
print "CCC", command
os.system(command)
os.system("git push")
else:
print "git commit not needed"
Shell.git("tag", "-a", v, "-m", "version {:}".format(v))
Shell.git("push", "origin", "--tags")
def rm(cls, cluster, id=None, format=None):
data = {
"CLUSTER": cluster,
"ID": id,
}
result = None
if id is not None:
try:
result = Shell.ssh(cluster, "rm -rf experiment/{ID}".format(**data))
except Exception, e:
pass
def tag(self):
v = self.find()
banner("v")
print v
#v = ".".join(self.version)
os.system("python shell_plugins.py install")
if self.git_commit_needed():
banner("git commit")
command = "git commit -m 'version {:}' {:}".format(
v, self.filename)
print "CCC", command
os.system(command)
os.system("git push")
else:
print "git commit not needed"
Shell.git("tag", "-a", v, "-m", "version {:}".format(v))
Shell.git("push", "origin", "--tags")
def list(cls, cluster, id=None, format=None):
data = {
"CLUSTER": cluster,
"ID": id,
}
result = None
if id is not None:
try:
result = Shell.ssh(cluster, "ls experiment/{ID}".format(**data))
result = result.split("\n")
except Exception, e:
result = None
def status(cls, host):
msg = "Unknown host"
try:
msg = Shell.ping("-c", "1", host)
except:
pass
if "1 packets transmitted, 1 packets received" in msg:
return True
elif "Unknown host" in msg:
return False
else:
return False
def status(cls, host):
msg = "Unknown host"
try:
msg = Shell.ping("-c", "1", host)
except:
pass
if "1 packets transmitted, 1 packets received" in msg:
return True
elif "Unknown host" in msg:
return False
else:
return False
def find_tunnel():
r = Shell.execute("ps", ["-ax"]).split("\n")
pid = None
info = None
for line in r:
if ("localhost" in line and "nucleus" in line) or ("comet" in line and "tunnel" in line) and not 'status' in line:
info = line.strip()
break
if info:
pid = int(info.split(" ", 1)[0])
return pid
def info(cls, cluster, format='json', all=False):
if all:
result = Shell.ssh(cluster, 'sinfo --format=\"%all\"')
else:
result = Shell.ssh(
cluster,
'sinfo --format=\"%P|%a|%l|%D|%t|%N\"')
# ignor leading lines till header is found
l = result.splitlines()
for i, res in enumerate(l):
if 'PARTITION|AVAIL|' in res:
result = "\n".join(l[i:])
break
parser = TableParser(strip=False)
d = parser.to_dict(result)
# add cluster and updated to each entry
for key in d.keys():
d[key]['cluster'] = cluster
d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if format == 'json':
return json.dumps(d, indent=4, separators=(',', ': '))
else:
return (dict_printer(d,
order=['cluster',
'partition',
'avail',
'timelimit',
'nodes',
'state',
'nodelist',
'updated'],
output=format))
def list_limits(cls, cloud, format, tenant=" "):
# set the environment variables
Quota.set_os_environment(cloud)
# execute the command
args = ["limits", "--tenant", tenant]
result = Shell.execute("nova", args)
# print results in a format
if "ERROR" in result:
return result
else:
d = Limits.convert_to_dict(result)
return dict_printer(d, order=["Name", "Used", "Max"], output=format)
def nodes(self, host, refresh=True):
"""
returns the information from the command pbsnodes in a dict.
:param host: the name of the host as specified in the .ssh/config file
:param refresh: if False, reads returns a cached value
if True, issues a new command and refreshes the cach
:return: information of the pbsnodes command in a dict
"""
manager_host = self.manager(host)
if self.pbs_nodes_data is None or refresh:
try:
result = Shell.ssh(manager_host, "pbsnodes", "-a")
except:
raise RuntimeError(
"can not execute pbs nodes on host {0}".format(
manager_host))
pbsinfo = {}
nodes = result.split("\n\n")
for node in nodes:
pbs_data = node.split("\n")
pbs_data = [e.strip() for e in pbs_data]
name = pbs_data[0]
if name != "":
pbsinfo[name] = {u'name': name}
for element in pbs_data[1:]:
try:
(attribute, value) = element.split(" = ")
if attribute == 'status':
status_elements = value.split(",")
pbsinfo[name][attribute] = {}
for e in status_elements:
(a, v) = e.split("=")
pbsinfo[name][attribute][a] = v
elif attribute == 'jobs':
pbsinfo[name][attribute] = value.split(',')
elif attribute == 'note' and (
value.strip().startswith("{")
or value.strip().startswith("[")):
pbsinfo[name][attribute] = literal_eval(value)
else:
pbsinfo[name][attribute] = value
except:
pass
self.pbs_nodes_data = pbsinfo
return self.pbs_nodes_data
def authors(self, output=None):
"""
returns the authors of the authors either as a text or as a dict. The
format is specified as an argument.
:param output: if "dict" is specified a dict will be returned
"""
result = Shell.git("shortlog", "-s", "-n")
if output is None:
return result
elif output == "dict":
authors = {}
for line in result.split("\n"):
l = " ".join(line.split()).strip()
(number, name) = l.split(" ", 1)
authors[name] = int(number)
return authors
def authors(self, output=None):
"""
returns the authors of the authors either as a text or as a dict. The
format is specified as an argument.
:param output: if "dict" is specified a dict will be returned
"""
result = Shell.git("shortlog", "-s", "-n")
if output is None:
return result
elif output == "dict":
authors = {}
for line in result.split("\n"):
l = " ".join(line.split()).strip()
(number, name) = l.split(" ", 1)
authors[name] = int(number)
return authors
def remote(cls, host, force=False):
"""
TODO: there is a bug in the instalation of kilo the openrc file on
the remote machine is not called openrc.sh but contains username and
project number.
:param host: the remote host
:param force:
:return:
"""
config = ConfigDict("cloudmesh.yaml")
host_spec = config["cloudmesh.clouds." + host]
host_credentials = host_spec["credentials"]
if 'cm_openrc' in host_spec:
Console.ok("looking for openrc")
else:
Console.error("no cm_openrc specified in the host")
return
hostname = config["cloudmesh.clouds." + host + ".cm_host"]
Console.ok("fetching information from {:} ...".format(host))
openrc = host_spec["cm_openrc"]
directory = os.path.dirname(openrc)
base = os.path.basename(openrc)
_from_dir = "{:}:{:}".format(hostname, directory).replace("~/", "")
_to_dir = os.path.dirname(Config.path_expand(directory))
openrc_file = Config.path_expand(openrc)
print("From: ", _from_dir)
print("To: ", _to_dir)
print("Openrc:", openrc_file)
cls.make_dir(_to_dir)
r = ""
Console.ok("Reading rc file from {}".format(host))
try:
r = Shell.scp('-r', _from_dir, _to_dir)
except Exception, e:
print(e)
return
def nodes(self, host, refresh=True):
"""
returns the information from the command pbsnodes in a dict.
:param host: the name of the host as specified in the .ssh/config file
:param refresh: if False, reads returns a cached value
if True, issues a new command and refreshes the cach
:return: information of the pbsnodes command in a dict
"""
manager_host = self.manager(host)
if self.pbs_nodes_data is None or refresh:
try:
result = Shell.ssh(manager_host, "pbsnodes", "-a")
except:
raise RuntimeError(
"can not execute pbs nodes on host {0}".format(manager_host))
pbsinfo = {}
nodes = result.split("\n\n")
for node in nodes:
pbs_data = node.split("\n")
pbs_data = [e.strip() for e in pbs_data]
name = pbs_data[0]
if name != "":
pbsinfo[name] = {u'name': name}
for element in pbs_data[1:]:
try:
(attribute, value) = element.split(" = ")
if attribute == 'status':
status_elements = value.split(",")
pbsinfo[name][attribute] = {}
for e in status_elements:
(a, v) = e.split("=")
pbsinfo[name][attribute][a] = v
elif attribute == 'jobs':
pbsinfo[name][attribute] = value.split(',')
elif attribute == 'note' and (
value.strip().startswith("{") or value.strip().startswith("[")):
pbsinfo[name][attribute] = literal_eval(value)
else:
pbsinfo[name][attribute] = value
except:
pass
self.pbs_nodes_data = pbsinfo
return self.pbs_nodes_data
def test_012_yaml_load(self):
"""
tests adding jobs from a YAML file
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
count_fgrep = len(Shell.fgrep("input:", "etc/jobs.yaml").split("\n"))
# Assert that the correct number jobs have been added
assert (db.count() == count_fgrep)
def test_15_jobscript(self):
HEADING()
db = self.db
db.connect()
name = "test1"
contents = "hallo"
db.add_script(name, contents)
db.add_script("test2", "other")
script = db.get_script(name)
print("Script:", script)
print("Content:", contents)
db.write_script(name, "/tmp/script.txt")
what = Shell.cat("/tmp/script.txt")
assert contents == what
assert contents == script
def jobstatus(self, host, jobid, kind='dict'):
"""
The status of a specific job
:param host: The host on which the job is running
:param jobid: The jobid as specified by the queing system
:param kind: The output can be returned as dict, xml, and yaml
:return:
"""
manager_host = self.manager(host)
qstat_xml_data = Shell.ssh(manager_host, "qstat", "-x", jobid).rstrip()
if kind == 'xml':
r = qstat_xml_data
else:
r = self.qstat_xml_to_dict(qstat_xml_data)
r[unicode(jobid)][u"cm_jobid"] = self.jobid
r[unicode(jobid)]["cm_Variable_list"] = self.variable_list(r)
if kind == 'yaml':
r = yaml.dump(r, default_flow_style=False)
return r
def start(self):
"""
starts the database service
:return: the pid
"""
if self.isup():
_pid = self.pid()
Console.error("A mongod process on port {:} is already running with pid {:}".format(self.port, _pid))
return
else:
Console.ok("STARTING")
try:
mongod = Shell.which("mongod")
command = [
mongod,
"--dbpath", str(self.db_path),
"--port", str(self.port),
"--fork",
"--logpath", str(self.log_file),
"--bind_ip", "127.0.0.1"
]
print(" ".join(command))
# a = subprocess.call(command)
os.system(" ".join(command))
self.ps()
Console.ok("MongoDB has been deployed")
self.info()
return None # implement the return of the pid for the process.
# store the pid in self.pid
except Exception, e:
Console.error("we had a problem starting the mongo daemon")
print(e)
Console.error("MongoDB has stopped")
# TODO remove the exit in final code, for debugging only
sys.exit()
def test_013_find_files(self):
"""
tests searching for a file
the file being searched for exists in 3 files: twice as input and twice as output
:return:
"""
HEADING()
db = self.db
db.connect()
# Clear all jobs currently in the database to ensure a correct final assertion
db.clear()
# Add the jobs outlined in the YAML file
db.add_from_yaml("etc/jobs.yaml")
inputs, outputs = db.find_jobs_with_file("in1.txt")
# Assert that the lengths of the inputs and outputs arrays are correct
count_fgrep = len(
Shell.fgrep("in1.txt", "etc/jobs.yaml").strip().split("\n"))
assert (len(inputs) == count_fgrep)
def qstat(self, host, user=True, format='dict'):
"""
executes the qstat command on a particular host and returns
the information as dict.
:param host: The host as specified in ~/.ssh/config
:param user: If True, only retirns information for the user
If False, all jobs for all users are returned
:param format:
:return:
"""
data = None
username = self.username(host)
manager_host = self.manager(host)
xml_data = Shell.ssh(manager_host, "qstat", "-x").rstrip()
if format == 'dict':
data = OpenPBS.qstat_xml_to_dict(xml_data)
selected_data = {}
for jobid in data:
(owner, cm_host) = data[jobid]['Job_Owner'].split('@')
if not user:
selected_data[unicode(jobid)] = data[unicode(jobid)]
elif owner == username:
selected_data[unicode(jobid)] = data[unicode(jobid)]
data = selected_data
for jobid in data:
data[unicode(jobid)][u"cm_jobid"] = jobid
if "Variable_list" in data[unicode(jobid)]:
data[unicode(
jobid)][u"cm_Variable_list"] = self.variable_list(
data, jobid)
elif format == "xml":
if user is not None:
print("WARNING: "
"restrictiong xml data for a user not supported.")
data = xml_data
return dict(data)
def ping(host):
"""ping the specified host.
:param host: the name or ip of the host
"""
try:
result = Shell.ping("-o", "-c", "1", host).strip().split("\n")
except:
pass
try:
(attributes, values) = result[-1].replace("round-trip", "")\
.strip().split("=")
attributes = attributes.strip().split("/")
values = values.strip().split("/")
data = dict(zip(attributes, values))
data['loss'] = result[-2].split(",")[2].split("%")[0].strip() + "%"
except:
data = {}
data['host'] = host
return data
def vbox_controlvm(*args):
a = ["controlvm"] + args
return Shell.execute('VBoxManage', a)
def vbox_vminfo(*args):
a = ["showvminfo"] + args
return Shell.execute('VBoxManage', a)
def vbox_list(*args):
a = ["list", "vms", "-l"] + args
return Shell.execute('VBoxManage', a)
def vbox_startcm(*args):
a = ["startvm"] + args
return Shell.execute('VBoxManage', a)
