def execute(self, name, command):
"""executes the command on the named host"""
if name in ["localhost"]:
r = '\n'.join(Shell.sh("-c", command).split()[-1:])
else:
r = '\n'.join(Shell.ssh(name, command).split()[-1:])
return r
def execute(self, name, command):
"""executes the command on the named host"""
if name in ["localhost"]:
r = '\n'.join(Shell.sh("-c", command).split()[-1:])
else:
r = '\n'.join(Shell.ssh(name, command).split()[-1:])
return r
def create_cloudmesh_yaml(filename):
if not os.path.exists(filename):
path = os.path.dirname(filename)
if not os.path.isdir(path):
Shell.mkdir(path)
etc_path = os.path.dirname(cloudmesh_client.__file__)
etc_file = os.path.join(etc_path, "etc", "cloudmesh.yaml")
to_dir = path_expand("~/.cloudmesh")
shutil.copy(etc_file, to_dir)
Console.ok("~/.cloudmesh/cloudmesh.yaml created")
def create_cloudmesh_yaml(filename):
if not os.path.exists(filename):
path = os.path.dirname(filename)
if not os.path.isdir(path):
Shell.mkdir(path)
etc_path = os.path.dirname(cloudmesh_client.__file__)
etc_file = os.path.join(etc_path, "etc", "cloudmesh.yaml")
to_dir = path_expand("~/.cloudmesh")
shutil.copy(etc_file, to_dir)
os.system("chmod -R go-rwx " + path_expand("~/.cloudmesh"))
Console.ok("~/.cloudmesh/cloudmesh.yaml created")
def delete(cls, cluster, job, group=None):
"""
This method is used to terminate a job with the specified or a group of jobs
job_id or job_name in a given cluster
:param group:
:param cluster: the cluster like comet
:param job: the job id or name
:return: success message or error
"""
try:
if group is not None:
# get the job ids from the db
arguments = {'cluster': cluster,
'group': group}
db_jobs = cls.cm.find('batchjob',
**arguments)
list1 = []
for i in db_jobs:
list1.append(db_jobs[i]['job_id'])
# read active jobs
active_jobs = json.loads(cls.queue(cluster))
list2 = []
for i in active_jobs:
list2.append(active_jobs[i]['jobid'])
# find intersection
res = set(list1).intersection(set(list2))
if res is not None:
for j in res:
cmd = 'scancel {}'.format(str(j))
Shell.ssh(cluster, cmd)
print("Deleted {}".format(j))
return "All jobs for group {} killed successfully".format(group)
else:
args = 'scancel '
if job.isdigit():
args += job
else:
args += "-n {}".format(job)
Shell.ssh(cluster, args)
return "Job {} killed successfully".format(job)
except Exception as ex:
print("in exceptio")
print(ex)
return ex
def _grep(self, search, platform):
if not search:
search = "'OS_PASSWORD': '******'"
cmd = "egrep -ri \"{0}\" * | cut -d\":\" -f1 > a.tmp".format(search)
print("[{0}]:{1}".format(platform, cmd))
os.system(cmd)
res = Shell.cat("a.tmp")
if res:
print ('[{0}]: [ERROR] PASSWORD(OR SECRET KEY) DETECTED, SEE FILES '
'BELOW'.format(platform))
print ("")
print (res)
else:
print ("[{0}]: NO PASSWORD DETECTED".format(platform))
Shell.rm("a.tmp")
print ("")
def list(cls, verbose=False):
def convert(line):
entry = (' '.join(line.split())).split(' ')
data = dotdict()
data.id = entry[0]
data.name = entry[1]
data.provider = entry[2]
data.state = entry[3]
data.directory = entry[4]
return data
result = Shell.execute("vagrant", "global-status --prune")
if verbose:
print(result)
if "There are no active" in result:
return None
lines = []
for line in result.split("\n")[2:]:
if line == " ":
break
else:
lines.append(convert(line))
return lines
def execute(cls, name, command, cwd=None):
vms = cls.to_dict(cls.list())
arg = "ssh {} -c {}".format(name, command)
result = Shell.execute("vagrant", ["ssh", name, "-c", command], cwd=vms[name]["directory"])
return result
def list(cls, verbose=False):
def convert(line):
entry = (' '.join(line.split())).split(' ')
data = dotdict()
data.id = entry[0]
data.name = entry[1]
data.provider = entry[2]
data.state = entry[3]
data.directory = entry[4]
return data
result = Shell.execute("vagrant", "global-status --prune")
if verbose:
print(result)
if "There are no active" in result:
return None
lines = []
for line in result.split("\n")[2:]:
if line == " ":
break
else:
lines.append(convert(line))
return lines
def _grep(self, search, platform):
if not search:
search = "'OS_PASSWORD': '******'"
cmd = "egrep -ri \"{0}\" * | cut -d\":\" -f1 > a.tmp".format(search)
print("[{0}]:{1}".format(platform, cmd))
os.system(cmd)
res = Shell.cat("a.tmp")
if res:
print('[{0}]: [ERROR] PASSWORD(OR SECRET KEY) DETECTED, SEE FILES '
'BELOW'.format(platform))
print("")
print(res)
else:
print("[{0}]: NO PASSWORD DETECTED".format(platform))
Shell.rm("a.tmp")
print("")
def emails(self, format_arg=None):
'''
returns the emails of the authors either as a text or as a dict. The
format is specified as an argument.
:param format_arg: if "dict" is specified a dict will be returned
:rtype: dict or arry of e-mails dependent on format_arg
'''
if format_arg is None:
format_string = "'%aN' <%cE>"
elif format_arg == 'dict':
format_string = "%aN\t%cE"
#result = Shell.sort(Shell.git("log",
# "--all",
# "--format=" + format_string,
# _tty_in=True,
# _tty_out=False,
# _piped=True), "-u")
result = Shell.execute("git",
["log", "--all", "--format=" + format_string])
if format_arg is None:
return result
elif format_arg == "dict":
result = iter(result.replace("\n", "\t").split("\t")[:-1])
emails = dict(zip(result, result))
for name in emails:
emails[name] = emails[name]
return emails
def run(command):
parameter = command.split(" ")
shell_command = parameter[0]
args = parameter[1:]
result = Shell.execute(shell_command, args)
print(result)
return result
def stat(self, email):
'''
returns a statistic of a git author with the given e_mail.
:param email: name of the author
:rtype: a dict with the statistics
'''
sums = [0, 0, 0]
for line in Shell.git("log",
"--all",
"--stat",
'--author={0}'.format(email),
_tty_in=True,
_tty_out=False,
_iter=True):
line = line[:-1]
if " files changed" in line:
line = line.replace(" insertions(+)", "")
line = line.replace(" insertion(+)", "")
line = line.replace(" deletion(-)", "")
line = line.replace(" deletions(-)", "")
line = line.replace(" files changed", "")
line = line.split(",")
data = [int(i) for i in line]
for index in range(0, len(data)):
sums[index] += data[index]
return {
"email": email,
"fileschanged": sums[0],
"inserted": sums[1],
"deleted": sums[2],
"lineschanged": sums[1] + sums[2]
}
def run(command):
print ("EXECUTING:", command)
parameter = command.split(" ")
shell_command = parameter[0]
args = parameter[1:]
result = Shell.execute(shell_command, args)
return result
def stop(self):
"""stops the mongo service."""
r = Shell.execute('mongod stop'.split(' '))
print(r)
# subprocess.call(mongod --shutdown)
log_print('stopped')
self.status()
def list(cls, cloud, start=None, end=None, tenant=None, format="table"):
# set the environment variables
set_os_environ(cloud)
try:
# execute the command
args = ["usage"]
if start is not None:
args.extend(["--start", start])
if end is not None:
args.extend(["--end", end])
if tenant is not None:
args.extend(["--tenant", tenant])
result = Shell.execute("nova", args)
result = Nova.remove_subjectAltName_warning(result)
lines = result.splitlines()
dates = lines[0]
# TODO: as stated below, nova returns additional lines,
# on my pc, SecurityWarning is returned, so filtering..
for l in lines[1:]:
if l.__contains__("SecurityWarning"):
lines.remove(l)
table = '\n'.join(lines[1:])
dates = dates.replace("Usage from ",
"").replace("to",
"").replace(" +",
" ")[:-1].split()
#
# TODO: for some reason the nova command has returned not the
# first + char, so we could not ignore the line we may set - as
# additional comment char, but that did not work
#
d = TableParser.convert(table, comment_chars="+#")
# d["0"]["start"] = "start"
# d["0"]["end"] = "end"
d["0"]["start"] = dates[0]
d["0"]["end"] = dates[1]
# del d['0']
return Printer.write(d,
order=[
"start", "end", "servers", "cpu hours",
"ram mb-hours", "disk gb-hours"
],
output=format)
except Exception as e:
return e
def setup(self, cloud="cloud=chameleon"):
self.cloud = cloud
print "Ignore Error: \n Please define a key first, e.g.: cm key add --ssh <keyname> \n " \
"-- If key has been successfully added into the database and uploaded into the cloud \n" \
"Ignore Error: \n problem uploading key veera to cloud chameleon: Key pair 'veera' already exists.\n " \
"******************************************************************************************************"
result = Shell.cm("reset")
print result
result = Shell.cm("key add --ssh")
print result
result = Shell.cm("key", "upload")
print result
result = Shell.cm("default", self.cloud)
print result
result = Shell.cm("refresh", "on")
print result
def run(command):
print(command)
parameter = command.split(" ")
shell_command = parameter[0]
args = parameter[1:]
result = Shell.execute(shell_command, args)
print(result)
return result
def do_delete(self, names):
names = str(names).split(' ')
for name in names:
delete_machine = "delete " + name
print delete_machine
result = Shell.cm("vm", delete_machine)
print result
def run(self, command):
command = command.format(**self.data)
banner(command)
parameter = command.split(" ")
shell_command = parameter[0]
args = parameter[1:]
result = Shell.execute(shell_command, args)
return result
def execute(cls, name, command, cwd=None):
vms = cls.to_dict(cls.list())
arg = "ssh {} -c {}".format(name, command)
result = Shell.execute("vagrant", ["ssh", name, "-c", command],
cwd=vms[name]["directory"])
return result
def list(cls, cloud, start=None, end=None, tenant=None, format="table"):
# set the environment variables
set_os_environ(cloud)
try:
# execute the command
args = ["usage"]
if start is not None:
args.extend(["--start", start])
if end is not None:
args.extend(["--end", end])
if tenant is not None:
args.extend(["--tenant", tenant])
result = Shell.execute("nova", args)
result = Nova.remove_subjectAltName_warning(result)
lines = result.splitlines()
dates = lines[0]
# TODO: as stated below, nova returns additional lines,
# on my pc, SecurityWarning is returned, so filtering..
for l in lines[1:]:
if l.__contains__("SecurityWarning"):
lines.remove(l)
table = '\n'.join(lines[1:])
dates = dates.replace("Usage from ", "").replace("to", "").replace(" +", " ")[:-1].split()
#
# TODO: for some reason the nova command has returned not the
# first + char, so we could not ignore the line we may set - as
# additional comment char, but that did not work
#
d = TableParser.convert(table, comment_chars="+#")
# d["0"]["start"] = "start"
# d["0"]["end"] = "end"
d["0"]["start"] = dates[0]
d["0"]["end"] = dates[1]
# del d['0']
return Printer.write(d,
order=["start",
"end",
"servers",
"cpu hours",
"ram mb-hours",
"disk gb-hours"],
output=format)
except Exception as e:
return e
def ec2(cls, cloud, zipfile):
def sanitize(name):
return name.replace(".zip", "").replace("@", "_")
def find_exports(filename):
with open(filename, "r") as f:
content = f.read()
data = {}
for line in content.split("\n"):
if line.startswith("export "):
line = line.replace("export ", "")
attribute, value = line.split("=", 1)
value = value.replace("${NOVA_KEY_DIR}/", "")
# remove comments
data[attribute] = value.split("#")[0].strip()
return data
base = sanitize(os.path.basename(zipfile))
dest = sanitize(os.path.join(
path_expand("~"),
".cloudmesh",
"clouds",
cloud,
os.path.basename(zipfile)))
Console.msg("Unzip file {} -> {}".format(zipfile, dest))
r = Shell.unzip(zipfile, dest)
rcfile = os.path.join(dest, "ec2rc.sh")
data = find_exports(rcfile)
data["DEST"] = dest
data["CLOUD"] = cloud
d = {
"cm_heading": "{CLOUD}, EC2".format(**data),
"cm_host": None,
"cm_label": "{CLOUD}_ec2".format(**data),
"cm_type": "ec2",
"cm_type_version": "ec2",
"credentials": {
"EC2_ACCESS_KEY": "{EC2_ACCESS_KEY}".format(**data),
"EC2_SECRET_KEY": "{EC2_SECRET_KEY}".format(**data),
"keyname": "TBD_not_used",
"userid": "TBD_not_used",
"EC2_URL": "{EC2_URL}".format(**data),
"EC2_USER_ID": "{EC2_USER_ID}",
"EC2_PRIVATE_KEY": "{DEST}/pk.pem".format(**data),
"EC2_CERT": "{DEST}/cert.pem".format(**data),
"NOVA_CERT": "{DEST}/cacert.pem".format(**data),
"EUCALYPTUS_CERT": "{DEST}/cacert.pem".format(**data),
},
"default": {
"flavor": "m1.small",
"image": "None",
}
}
config = ConfigDict("cloudmesh.yaml")
config["cloudmesh"]["clouds"][cloud] = d
config.save()
def rm(cls, cluster, id=None, format=None):
data = {
"CLUSTER": cluster,
"ID": id,
}
result = None
if id is not None:
try:
result = Shell.ssh(cluster, "rm -rf experiment/{ID}".format(**data))
except Exception as e:
pass
else:
try:
result = Shell.ssh(cluster, "rm -rf experiment/*").split("\n")
except Exception as e:
pass
return result
def run(self, command):
command = command.format(**self.data)
banner(command, c='-')
parameter = command.split(" ")
shell_command = parameter[0]
args = parameter[1:]
result = Shell.execute(shell_command, args)
print(result)
return str(result)
def rm(cls, cluster, id=None, format=None):
data = {
"CLUSTER": cluster,
"ID": id,
}
result = None
if id is not None:
try:
result = Shell.ssh(cluster,
"rm -rf experiment/{ID}".format(**data))
except Exception as e:
pass
else:
try:
result = Shell.ssh(cluster, "rm -rf experiment/*").split("\n")
except Exception as e:
pass
return result
def version(verbose=False):
result = Shell.execute("vagrant", ["version"])
if verbose:
return result
else:
lines = result.split("\n")
for line in lines:
if "Installed Version:" in line:
return line.replace("Installed Version:", "").strip()
return None
def sync(cls, cloudname, localdir, remotedir, operation=None):
"""
Syncs a local directory with a remote directory.
Either from local to remote OR vice-versa
:param cloudname:
:param localdir:
:param remotedir:
:param operation: get/put
:return:
"""
# Get the operating system
os_type = cls.operating_system()
# fix the local dir path
localdirpath = Config.path_expand(localdir)
# check if local directory exists
if not os.path.exists(localdirpath):
if operation == "put":
Console.error("The local directory [{}] does not exist."
.format(localdirpath))
return None
elif operation == "get":
# for receiving, create local dir
os.mkdir(localdirpath)
Console.msg("Created local directory [{}] for sync."
.format(localdirpath))
"""
rsync now works on windows machines as well.
we install rsync (v5.4.1.20150827) on windows via chocolatey
$ choco install rsync
"""
host = cls.get_host(cloudname)
if host is None:
Console.error("Cloud [{}] not found in cloudmesh.yaml file."
.format(cloudname))
return None
else:
args = None
if operation == "put":
args = [
"-r",
localdir,
host + ":" + remotedir
]
elif operation == "get":
args = [
"-r",
host + ":" + remotedir,
localdir
]
# call rsync
return Shell.rsync(*args)
def run(self, command):
self.cloud = "aws"
command = command.format(**self.data)
banner(command, c="-")
print(command)
parameter = command.split(" ")
shell_command = parameter[0]
args = parameter[1:]
result = Shell.execute(shell_command, args)
print(result)
return result
def info(cls, name=None):
result = Shell.execute("vagrant", ["ssh-config"], cwd=name)
lines = result.split("\n")
data = {}
for line in lines:
attribute, value = line.strip().split(" ", 1)
if attribute == "IdentityFile":
value = value.replace('"', '')
data[attribute] = value
return data
def run(self, command):
self.cloud = "azure"
command = command.format(**self.data)
banner(command, c="-")
print(command)
parameter = command.split(" ")
shell_command = parameter[0]
args = parameter[1:]
result = Shell.execute(shell_command, args)
print(result)
return result
def pid(self):
process_id = None
output = Shell.ps('-ax')
for line in output.split("\n"):
if 'mongod' in line and "--port" in line:
log_print(line)
process_id = line.split(" ")[0]
return process_id
return process_id
def ec2(cls, cloud, zipfile):
def sanitize(name):
return name.replace(".zip", "").replace("@", "_")
def find_exports(filename):
with open(filename, "r") as f:
content = f.read()
data = {}
for line in content.split("\n"):
if line.startswith("export "):
line = line.replace("export ", "")
attribute, value = line.split("=", 1)
value = value.replace("${NOVA_KEY_DIR}/", "")
# remove comments
data[attribute] = value.split("#")[0].strip()
return data
base = sanitize(os.path.basename(zipfile))
dest = sanitize(
os.path.join(path_expand("~"), ".cloudmesh", "clouds", cloud,
os.path.basename(zipfile)))
Console.msg("Unzip file {} -> {}".format(zipfile, dest))
r = Shell.unzip(zipfile, dest)
rcfile = os.path.join(dest, "ec2rc.sh")
data = find_exports(rcfile)
data["DEST"] = dest
data["CLOUD"] = cloud
d = {
"cm_heading": "{CLOUD}, EC2".format(**data),
"cm_host": None,
"cm_label": "{CLOUD}_ec2".format(**data),
"cm_type": "ec2",
"cm_type_version": "ec2",
"credentials": {
"EC2_ACCESS_KEY": "{EC2_ACCESS_KEY}".format(**data),
"EC2_SECRET_KEY": "{EC2_SECRET_KEY}".format(**data),
"keyname": "TBD_not_used",
"userid": "TBD_not_used",
"EC2_URL": "{EC2_URL}".format(**data),
"EC2_USER_ID": "{EC2_USER_ID}",
"EC2_PRIVATE_KEY": "{DEST}/pk.pem".format(**data),
"EC2_CERT": "{DEST}/cert.pem".format(**data),
"NOVA_CERT": "{DEST}/cacert.pem".format(**data),
"EUCALYPTUS_CERT": "{DEST}/cacert.pem".format(**data),
},
"default": {
"flavor": "m1.small",
"image": "None",
}
}
from pprint import pprint
config = ConfigDict("cloudmesh.yaml")
config["cloudmesh"]["clouds"][cloud] = d
config.save()
def status(cls, host):
msg = "Unknown host"
try:
msg = Shell.ping("-c", "1", host)
except:
pass
if "1 packets transmitted, 1 packets received" in msg:
return True
elif "Unknown host" in msg:
return False
else:
return False
def find_tunnel():
r = Shell.execute("ps", ["-ax"]).split("\n")
pid = None
info = None
for line in r:
if ("localhost" in line and "nucleus" in line) or ("comet" in line and "tunnel" in line) and not 'status' in line:
info = line.strip()
break
if info:
pid = int(info.split(" ", 1)[0])
return pid
def status(cls, host):
msg = "Unknown host"
try:
msg = Shell.ping("-c", "1", host)
except:
pass
if "1 packets transmitted, 1 packets received" in msg:
return True
elif "Unknown host" in msg:
return False
else:
return False
def do_version(self, args, arguments):
"""
Usage:
version [--format=FORMAT] [--check=CHECK]
Options:
--format=FORMAT the format to print the versions in [default: table]
--check=CHECK boolean tp conduct an additional check [default: True]
Description:
Prints out the version number
"""
python_version, pip_version = get_python()
try:
git_hash_version = Shell.execute('git',
'log -1 --format=%h',
traceflag=False,
witherror=False)
except:
git_hash_version = 'N/A'
versions = {
"cloudmesh_client": {
"name": "cloudmesh_client",
"version": str(cloudmesh_client.__version__)
},
# "cloudmesh_base": {
# "name": "cloudmesh_base",
# "version": str(cloudmesh_base.__version__)
# },
"python": {
"name": "python",
"version": str(python_version)
},
"pip": {
"name": "pip",
"version": str(pip_version)
},
"git": {
"name": "git hash",
"version": str(git_hash_version)
}
}
print(
Printer.write(versions,
output=arguments["--format"],
order=["name", "version"]))
if arguments["--check"] in ["True"]:
check_python()
def info(cls, name=None):
result = Shell.execute("vagrant",
["ssh-config"],
cwd=name)
lines = result.split("\n")
data = {}
for line in lines:
attribute, value = line.strip().split(" ", 1)
if attribute == "IdentityFile":
value = value.replace('"','')
data[attribute] = value
return data
def find_tunnel():
r = Shell.execute("ps", ["-ax"]).split("\n")
pid = None
info = None
for line in r:
if ("localhost" in line and "nucleus" in line) or (
"comet" in line and "tunnel" in line) and 'status' not in line:
info = line.strip()
break
if info:
pid = int(info.split(" ", 1)[0])
return pid
def list(cls, cluster, id=None, format=None):
data = {
"CLUSTER": cluster,
"ID": id,
}
result = None
if id is not None:
try:
result = Shell.ssh(cluster, "ls experiment/{ID}".format(**data))
result = result.split("\n")
except Exception as e:
result = None
else:
try:
result = Shell.ssh(cluster, "ls experiment").split("\n")
ids = sorted([int(i) for i in result])
if format not in [None, 'txt']:
result = ids
except Exception as e:
result = None
return result
def info(cls, cluster, format='json', all=False):
if all:
result = Shell.ssh(cluster, 'sinfo --format=\"%all\"')
else:
result = Shell.ssh(
cluster,
'sinfo --format=\"%P|%a|%l|%D|%t|%N\"')
# ignore leading lines till header is found
l = result.splitlines()
for i, res in enumerate(l):
if 'PARTITION|AVAIL|' in res:
result = "\n".join(l[i:])
break
parser = TableParser(strip=False)
d = parser.to_dict(result)
# add cluster and updated to each entry
for key in list(d.keys()):
d[key]['cluster'] = cluster
d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if format == 'json':
return json.dumps(d, indent=4, separators=(',', ': '))
else:
return (Printer.write(d,
order=['cluster',
'partition',
'avail',
'timelimit',
'nodes',
'state',
'nodelist',
'updated'],
output=format))
def list(cls, cluster, id=None, format=None):
data = {
"CLUSTER": cluster,
"ID": id,
}
result = None
if id is not None:
try:
result = Shell.ssh(cluster,
"ls experiment/{ID}".format(**data))
result = result.split("\n")
except Exception as e:
result = None
else:
try:
result = Shell.ssh(cluster, "ls experiment").split("\n")
ids = sorted([int(i) for i in result])
if format not in [None, 'txt']:
result = ids
except Exception as e:
result = None
return result
def do_version(self, args, arguments):
"""
Usage:
version [--format=FORMAT] [--check=CHECK]
Options:
--format=FORMAT the format to print the versions in [default: table]
--check=CHECK boolean tp conduct an additional check [default: True]
Description:
Prints out the version number
"""
python_version, pip_version = get_python()
try:
git_hash_version = Shell.execute('git', 'log -1 --format=%h')
except:
git_hash_version = 'N/A'
versions = {
"cloudmesh_client": {
"name": "cloudmesh_client",
"version": str(cloudmesh_client.__version__)
},
# "cloudmesh_base": {
# "name": "cloudmesh_base",
# "version": str(cloudmesh_base.__version__)
# },
"python": {
"name": "python",
"version": str(python_version)
},
"pip": {
"name": "pip",
"version": str(pip_version)
},
"git": {
"name": "git hash",
"version": str(git_hash_version)
}
}
print(dict_printer(versions, output=arguments["--format"],
order=["name", "version"]))
if arguments["--check"] in ["True"]:
check_python()
def sync(cls, cloudname, localdir, remotedir, operation=None):
"""
Syncs a local directory with a remote directory.
Either from local to remote OR vice-versa
:param cloudname:
:param localdir:
:param remotedir:
:param operation: get/put
:return:
"""
# Get the operating system
os_type = cls.operating_system()
# fix the local dir path
localdirpath = Config.path_expand(localdir)
# check if local directory exists
if not os.path.exists(localdirpath):
if operation == "put":
Console.error(
"The local directory [{}] does not exist.".format(
localdirpath))
return None
elif operation == "get":
# for receiving, create local dir
os.mkdir(localdirpath)
Console.msg("Created local directory [{}] for sync.".format(
localdirpath))
"""
rsync now works on windows machines as well.
we install rsync (v5.4.1.20150827) on windows via chocolatey
$ choco install rsync
"""
host = cls.get_host(cloudname)
if host is None:
Console.error(
"Cloud [{}] not found in cloudmesh.yaml file.".format(
cloudname))
return None
else:
args = None
if operation == "put":
args = ["-r", localdir, host + ":" + remotedir]
elif operation == "get":
args = ["-r", host + ":" + remotedir, localdir]
# call rsync
return Shell.rsync(*args)
def queue(cls, cluster, format='json', job=None):
try:
args = 'squeue '
if job is not None:
if job.isdigit():
args += ' -j {} '.format(str(job)) # search by job id
else:
args += ' -n {} '.format(job) # search by job name
f = '--format=%all'
args += f
result = Shell.ssh(cluster, args)
# TODO: process till header is found...(Need a better way)
l = result.splitlines()
for i, res in enumerate(l):
if 'ACCOUNT|GRES|' in res:
result = "\n".join(str(x) for x in l[i:])
break
parser = TableParser(strip=True)
d = parser.to_dict(result)
# add cluster and updated to each entry
for key in list(d.keys()):
d[key]['cluster'] = cluster
d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if format == 'json':
return json.dumps(d, indent=4, separators=(',', ': '))
else:
return (Printer.write(d,
order=['cluster',
'jobid',
'partition',
'name',
'user',
'st',
'time',
'nodes',
'nodelist',
'updated'],
output=format))
except Exception as e:
Error.traceback(e)
return e
def list(cls):
def convert(line):
line = line.replace("(", "")
line = line.replace(")", "")
line = line.replace(",", "")
entry = line.split(" ")
data = dotdict()
data.name = entry[0]
data.provider = entry[1]
data.date = entry[2]
return data
result = Shell.execute("vagrant", ["box", "list"])
lines = []
for line in result.split("\n"):
lines.append(convert(line))
return lines
def authors(self, format_arg=None):
'''
returns the authors of the authors either as a text or as a dict. The
format is specified as an argument.
:param format_arg: if "dict" is specified a dict will be returned
'''
#result = Shell.git("shortlog", "-s", "-n", _tty_in=True, _tty_out=False)
result = Shell.execute("git", ["shortlog", "-s", "-n"])
if format_arg is None:
return result
elif format_arg == "dict":
list_string = result.replace("\n", "\t").split("\t")[:-1]
it = iter(list_string[::-1])
authors = dict(zip(it, it))
for name in authors:
authors[name] = int(authors[name])
return authors
def output(cls, cluster, id=None, format=None):
data = {
"CLUSTER": cluster,
"ID": id,
}
result = None
if id is None:
ids = list(cluster)
else:
ids = [id]
result = []
for i in ids:
try:
result.append(Shell.ssh(cluster, "cat experiment/{}/*.out".format(i)))
except:
result.append("")
# if result == []:
# result = None
return result
def output(cls, cluster, id=None, format=None):
data = {
"CLUSTER": cluster,
"ID": id,
}
result = None
if id is None:
ids = list(cluster)
else:
ids = [id]
result = []
for i in ids:
try:
result.append(
Shell.ssh(cluster, "cat experiment/{}/*.out".format(i)))
except:
result.append("")
# if result == []:
# result = None
return result
def boot(cls, **kwargs):
arg = dotdict(kwargs)
arg.cwd = kwargs.get("cwd", None)
vms = cls.to_dict(cls.list())
if arg.name in vms:
Console.error("vm {name} already booted".format(**arg),
traceflag=False)
return None
# print result
else:
cls.create(**kwargs)
Console.ok("{name} created".format(**arg))
Console.ok("{name} booting ...".format(**arg))
result = Shell.execute("vagrant", ["up", arg.name], cwd=arg.name)
Console.ok("{name} ok.".format(**arg))
return result
def boot(cls, **kwargs):
arg = dotdict(kwargs)
arg.cwd = kwargs.get("cwd", None)
vms = cls.to_dict(cls.list())
if arg.name in vms:
Console.error("vm {name} already booted".format(**arg), traceflag=False)
return None
# print result
else:
cls.create(**kwargs)
Console.ok("{name} created".format(**arg))
Console.ok("{name} booting ...".format(**arg))
result = Shell.execute("vagrant",
["up", arg.name],
cwd=arg.name)
Console.ok("{name} ok.".format(**arg))
return result
def add(cls, name):
result = Shell.execute("vagrant", ["box", "add", name])
return result
def run(cls, cluster, group, cmd, **kwargs):
# determine the script name..
#
# TODO: script count is variable in data base, we test if fil exists and if it
# does increase counter till we find one that does not, that will be new counter.
# new counter will than be placed in db.
#
# define get_script_name(directory, prefix, counter)
# there maybe s a similar thing already in the old cloudmesh
#
# if not kwargs['-name']:
#
# old_count = Shell.ssh(cluster,
# "ls {}*.sh | wc -l | sed 's/$/ count/'".
# format(username))
# c = [f for f in old_count.splitlines() if 'count' in f]
# script_count = c[0].split()[0]
# else:
# script_count = kwargs['-name']
config = cls.read_config(cluster)
if config["credentials"]["username"] == 'TBD':
return "Please enter username in cloudmesh.yaml for cluster {}".format(cluster)
cls.incr()
data = {
"cluster": cluster,
"count": cls.counter(),
"username": config["credentials"]["username"],
"remote_experiment_dir": config["default"]["experiment_dir"],
"queue": config["default"]["queue"],
"id": None,
"nodes": 1,
"tasks_per_node": 1,
}
data["script_base_name"] = "{username}-{count}".format(**data)
data["script_name"] = "{username}-{count}.sh".format(**data)
data["script_output"] = "{username}-{count}.out".format(**data)
data["script_error"] = "{username}-{count}.err".format(**data)
data["remote_experiment_dir"] = \
"{remote_experiment_dir}/{count}".format(**data).format(**data)
data["group"] = group
# overwrite defaults
option_mapping = {'-t': '{tasks_per_node}'.format(**data),
'-N': '{nodes}'.format(**data),
'-p': '{queue}'.format(**data),
'-o': '{script_output}'.format(**data),
'-D': '{remote_experiment_dir}'.format(**data),
'-e': '{script_error}'.format(**data)}
# map(lambda k, v:
# option_mapping.__setitem__(k, kwargs.get(k) or v),
# option_mapping.items())
#
# rewrite for better readability
for (k, v) in iteritems(option_mapping):
option_mapping[k] = kwargs.get(k) or v
config = cls.read_config(cluster)
project = None
try:
project = config["credentials"]["project"]
if project.lower() not in ["tbd", "none"]:
option_mapping["-A"] = project
except:
pass
for key in option_mapping:
data[key] = option_mapping[key]
# create the options for the script
options = ""
for key, value in option_mapping.items():
options += '#SBATCH {} {}\n'.format(key, value)
cls.create_remote_dir(cluster, data["remote_experiment_dir"])
# if the command is a script, copy the script
if os.path.isfile(Config.path_expand(cmd)):
_from = Config.path_expand(cmd)
_to = '{cluster}:{remote_experiment_dir}'.format(**data)
local_file_name = cmd.split('/')[-1]
Shell.execute("rsync", [_from, _to])
data["command"] = '{remote_experiment_dir}/{local_file_name}'.format(local_file_name=local_file_name,
**data)
else:
data["command"] = cmd
data["options"] = options
script = textwrap.dedent(
"""
#! /bin/sh
{options}
echo '#CLOUDMESH: BATCH ENVIRONMENT'
echo 'BASIL_RESERVATION_ID:' $BASIL_RESERVATION_ID
echo 'SLURM_CPU_BIND:' $SLURM_CPU_BIND
echo 'SLURM_JOB_ID:' $SLURM_JOB_ID
echo 'SLURM_JOB_CPUS_PER_NODE:' $SLURM_JOB_CPUS_PER_NODE
echo 'SLURM_JOB_DEPENDENCY:' $SLURM_JOB_DEPENDENCY
echo 'SLURM_JOB_NAME:' $SLURM_JOB_NAME
echo 'SLURM_JOB_NODELIST:' $SLURM_JOB_NODELIST
echo 'SLURM_JOB_NUM_NODES:' $SLURM_JOB_NUM_NODES
echo 'SLURM_MEM_BIND:' $SLURM_MEM_BIND
echo 'SLURM_TASKS_PER_NODE:' $SLURM_TASKS_PER_NODE
echo 'MPIRUN_NOALLOCATE:' $MPIRUN_NOALLOCATE
echo 'MPIRUN_NOFREE:' $MPIRUN_NOFREE
echo 'SLURM_NTASKS_PER_CORE:' $SLURM_NTASKS_PER_CORE
echo 'SLURM_NTASKS_PER_NODE:' $SLURM_NTASKS_PER_NODE
echo 'SLURM_NTASKS_PER_SOCKET:' $SLURM_NTASKS_PER_SOCKET
echo 'SLURM_RESTART_COUNT:' $SLURM_RESTART_COUNT
echo 'SLURM_SUBMIT_DIR:' $SLURM_SUBMIT_DIR
echo 'MPIRUN_PARTITION:' $MPIRUN_PARTITION
d=$(date)
echo \"#CLOUDMESH: status, start, $d\"
srun -l echo \"#CLOUDMESH: status, start, $d\"
srun -l {command}
d=$(date)
srun -l echo \"#CLOUDMESH: status, finished, $d\"
d=$(date)
echo \"#CLOUDMESH: status, finished, $d\"
"""
).format(**data).replace("\r\n", "\n").strip()
_from = Config.path_expand('~/.cloudmesh/{script_name}'.format(**data))
_to = '{cluster}:{remote_experiment_dir}'.format(**data)
data["from"] = _from
data["to"] = _to
data["script"] = script
# write the script to local
# print(_from)
# print(_to)
with open(_from, 'w') as local_file:
local_file.write(script)
# copy to remote host
Shell.scp(_from, _to)
# delete local file
# Shell.execute('rm', _from)
# import sys; sys.exit()
# run the sbatch command
cmd = 'sbatch {remote_experiment_dir}/{script_name}'.format(**data)
data["cmd"] = cmd
# print ("CMD>", cmd)
result = Shell.ssh(cluster, cmd)
data["output"] = result
# find id
for line in result.split("\n"):
# print ("LLL>", line)
if "Submitted batch job" in line:
data["job_id"] = int(line.replace("Submitted batch job ", "").strip())
break
#
# HACK, should not depend on Model.py
#
# from cloudmesh_client.db.model import BATCHJOB
# name = ""
# BATCHJOB(name,
# cluster=data["cluster"],
# id=data["id"],
# script=data["script"]) # has user and username which seems wrong
# here what we have in data and want to store the - options are obviously wrong
# and need to be full names
# noinspection PyPep8,PyPep8
"""
{'-D': '/N/u/gvonlasz/experiment/3',
'-N': '1',
'-o': 'gvonlasz-3.out',
'-p': 'delta',
'-t': '1',
'cluster': 'india',
'cmd': 'sbatch /N/u/gvonlasz/experiment/3/gvonlasz-3.sh',
'command': 'uname',
'count': 3,
'from': '/Users/big/.cloudmesh/gvonlasz-3.sh',
'id': 1346,
'options': '#SBATCH -t 1\n#SBATCH -o gvonlasz-3.out\n#SBATCH -N 1\n#SBATCH -p delta\n#SBATCH -D /N/u/gvonlasz/experiment/3\n',
'output': 'Submitted batch job 1346',
'queue': 'delta',
'remote_experiment_dir': '/N/u/gvonlasz/experiment/3',
'script': "#! /bin/sh\n#SBATCH -t 1\n#SBATCH -o gvonlasz-3.out\n#SBATCH -N 1\n#SBATCH -p delta\n#SBATCH -D /N/u/gvonlasz/experiment/3\n\nsrun -l echo '#CLOUDMESH: Starting'\nsrun -l uname\nsrun -l echo '#CLOUDMESH: Test ok'",
'script_base_name': 'gvonlasz-3',
'script_name': 'gvonlasz-3.sh',
'script_output': 'gvonlasz-3.out',
'to': 'india:/N/u/gvonlasz/experiment/3',
'username': '******'}
"""
"""
we also want to store what part of the .out file,
BASIL_RESERVATION_ID:
SLURM_CPU_BIND:
SLURM_JOB_ID: 1351
SLURM_JOB_CPUS_PER_NODE: 12
SLURM_JOB_DEPENDENCY:
SLURM_JOB_NAME: gvonlasz-8.sh
SLURM_JOB_NODELIST: d001
SLURM_JOB_NUM_NODES: 1
SLURM_MEM_BIND:
SLURM_TASKS_PER_NODE: 12
MPIRUN_NOALLOCATE:
MPIRUN_NOFREE:
SLURM_NTASKS_PER_CORE:
SLURM_NTASKS_PER_NODE:
SLURM_NTASKS_PER_SOCKET:
SLURM_RESTART_COUNT:
SLURM_SUBMIT_DIR: /N/u/gvonlasz
MPIRUN_PARTITION:
so maybe we want to use some of the names here as they reflect the env vars
"""
#
# add data to database
#
# remove the - options
for key in ['-t', '-N', '-p', '-o', '-D', '-e']:
if key in data:
print(key, data[key])
del data[key]
data['status'] = 'started'
cls.add_db(**data)
return data
def test(cls, cluster, time):
result = Shell.ssh(cluster,
"srun -n1 -t {} echo '#CLOUDMESH: Test ok'".format(
time))
return result
def do_nova(self, args, arguments):
"""
::
Usage:
nova set CLOUD
nova info [CLOUD] [--password]
nova help
nova [--group=GROUP] ARGUMENTS...
A simple wrapper for the openstack nova command
Arguments:
GROUP The group to add vms to
ARGUMENTS The arguments passed to nova
help Prints the nova manual
set reads the information from the current cloud
and updates the environment variables if
the cloud is an openstack cloud
info the environment values for OS
Options:
--group=GROUP Add VM to GROUP group
--password Prints the password
-v verbose mode
"""
# pprint(arguments)
cloud = arguments['CLOUD'] or Default.cloud
if not cloud:
Console.error("Default cloud not set!")
return ""
group = arguments["--group"] or Default.group
if not group:
Console.error("Default group not set!")
return ""
if arguments["help"]:
os.system("nova help")
return ""
elif arguments["info"]:
set_os_environ(cloud)
d = {}
#
# TODO: this naturally does not work as clouds will have
# different parameters. ALos it does not unset previous
# parameters from other clouds. See register
#
for attribute in ['OS_USERNAME',
'OS_TENANT_NAME',
'OS_AUTH_URL',
'OS_CACERT',
'OS_PASSWORD',
'OS_REGION']:
try:
d[attribute] = os.environ[attribute]
except:
Console.warning("OS environment variable {:} not found"
.format(attribute))
d[attribute] = None
if not arguments["--password"]:
d['OS_PASSWORD'] = "******"
print(Printer.row_table(d, order=None, labels=["Variable", "Value"]))
msg = "info. OK."
Console.ok(msg)
return ""
elif arguments["set"]:
if cloud:
set_os_environ(cloud)
msg = "{0} is set".format(cloud)
Console.ok(msg)
else:
Console.error("CLOUD is required")
else: # nova ARGUMENTS...
print("Cloud = {0}".format(cloud))
try:
set_os_environ(cloud)
args = arguments["ARGUMENTS"]
# arguments may contain multiple optional arguments
if len(args) == 1:
args = args[0].split()
result = Shell.execute("nova", args)
print (result)
# print(Nova.remove_subjectAltName_warning(result))
"""
If request for nova boot,
add the vm to group specified,
or else add to default group
"""
if "boot" in args:
# Logic to find ID of VM in the result
fields = []
for field in result.split("|"):
fields.append(field.strip())
index = fields.index('id') + 1
vm_id = fields[index]
# Add to group
Group.add(name=group, species="vm", member=vm_id, category=cloud)
except Exception as ex:
Console.error("Error executing Nova command: {}".format(ex))
return ""
def do_vm(self, args, arguments):
"""
::
Usage:
vm default [--cloud=CLOUD][--format=FORMAT]
vm refresh [all][--cloud=CLOUD]
vm boot [--name=NAME]
[--cloud=CLOUD]
[--username=USERNAME]
[--image=IMAGE]
[--flavor=FLAVOR]
[--group=GROUP]
[--public]
[--secgroup=SECGROUP]
[--key=KEY]
[--dryrun]
vm boot [--n=COUNT]
[--cloud=CLOUD]
[--username=USERNAME]
[--image=IMAGE]
[--flavor=FLAVOR]
[--group=GROUP]
[--public]
[--secgroup=SECGROUP]
[--key=KEY]
[--dryrun]
vm ping [NAME] [N]
vm console [NAME]
[--group=GROUP]
[--cloud=CLOUD]
[--force]
vm start [NAMES]
[--group=GROUP]
[--cloud=CLOUD]
[--force]
vm stop [NAMES]
[--group=GROUP]
[--cloud=CLOUD]
[--force]
vm terminate [NAMES]
[--group=GROUP]
[--cloud=CLOUD]
[--force]
vm delete [NAMES]
[--group=GROUP]
[--cloud=CLOUD]
[--keep]
[--dryrun]
vm ip assign [NAMES]
[--cloud=CLOUD]
vm ip show [NAMES]
[--group=GROUP]
[--cloud=CLOUD]
[--format=FORMAT]
[--refresh]
vm ip inventory [NAMES]
[--header=HEADER]
[--file=FILE]
vm ssh [NAME] [--username=USER]
[--quiet]
[--ip=IP]
[--cloud=CLOUD]
[--key=KEY]
[--command=COMMAND]
vm rename [OLDNAMES] [NEWNAMES] [--force] [--dryrun]
vm list [NAMES]
[--cloud=CLOUDS|--active]
[--group=GROUP]
[--format=FORMAT]
[--refresh]
vm status [NAMES]
vm wait [--cloud=CLOUD] [--interval=SECONDS]
vm info [--cloud=CLOUD]
[--format=FORMAT]
vm check NAME
vm username USERNAME [NAMES] [--cloud=CLOUD]
Arguments:
COMMAND positional arguments, the commands you want to
execute on the server(e.g. ls -a) separated by ';',
you will get a return of executing result instead of login to
the server, note that type in -- is suggested before
you input the commands
NAME server name. By default it is set to the name of last vm from database.
NAMES server name. By default it is set to the name of last vm from database.
KEYPAIR_NAME Name of the openstack keypair to be used to create VM. Note this is
not a path to key.
NEWNAMES New names of the VM while renaming.
OLDNAMES Old names of the VM while renaming.
Options:
--username=USERNAME the username to login into the vm. If not specified it will be guessed
from the image name and the cloud
--ip=IP give the public ip of the server
--cloud=CLOUD give a cloud to work on, if not given, selected
or default cloud will be used
--count=COUNT give the number of servers to start
--detail for table print format, a brief version
is used as default, use this flag to print
detailed table
--flavor=FLAVOR give the name or id of the flavor
--group=GROUP give the group name of server
--secgroup=SECGROUP security group name for the server
--image=IMAGE give the name or id of the image
--key=KEY specify a key to use, input a string which
is the full path to the private key file
--keypair_name=KEYPAIR_NAME Name of the openstack keypair to be used to create VM.
Note this is not a path to key.
--user=USER give the user name of the server that you want
to use to login
--name=NAME give the name of the virtual machine
--force rename/ delete vms without user's confirmation
--command=COMMAND
specify the commands to be executed
Description:
commands used to boot, start or delete servers of a cloud
vm default [options...]
Displays default parameters that are set for vm boot either on the
default cloud or the specified cloud.
vm boot [options...]
Boots servers on a cloud, user may specify flavor, image .etc, otherwise default values
will be used, see how to set default values of a cloud: cloud help
vm start [options...]
Starts a suspended or stopped vm instance.
vm stop [options...]
Stops a vm instance .
vm delete [options...]
Delete servers of a cloud, user may delete a server by its name or id, delete servers
of a group or servers of a cloud, give prefix and/or range to find servers by their names.
Or user may specify more options to narrow the search
vm floating_ip_assign [options...]
assign a public ip to a VM of a cloud
vm ip show [options...]
show the ips of VMs
vm ssh [options...]
login to a server or execute commands on it
vm list [options...]
same as command "list vm", please refer to it
vm status [options...]
Retrieves status of last VM booted on cloud and displays it.
Tip:
give the VM name, but in a hostlist style, which is very
convenient when you need a range of VMs e.g. sample[1-3]
=> ['sample1', 'sample2', 'sample3']
sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18']
Quoting commands:
cm vm login gvonlasz-004 --command=\"uname -a\"
"""
"""
# terminate
# issues a termination to the cloud, keeps vm in database
# delete
# issues a terminate if not already done
# (remember you do not have to go to cloud if state is already terminated)
# deletes the vm from database
#
# bulk rename
rename abc[0-1] def[3-4] renames the abc0,abc1 -> def3,def4
if arguments["rename"]:
oldnames = Parameter.expand(arguments["OLDNAME"])
newnames = Parameter.expand(arguments["NEWNAME"])
# check if new names ar not already taken
# to be implemented
if len(oldnames) == len(newnames):
for i in range(0, len(oldnames)):
oldname = oldnames[i]
newname = newnames[i]
if newname is None or newname == '':
print("New node name cannot be empty")
else:
print(Cluster.rename_node(clusterid, oldname, newname))
"""
cm = CloudmeshDatabase()
def _print_dict(d, header=None, output='table'):
return Printer.write(d, order=["id", "name", "status"], output=output, sort_keys=True)
def _print_dict_ip(d, header=None, output='table'):
return Printer.write(d, order=["network", "version", "addr"], output=output, sort_keys=True)
def get_vm_name(name=None, offset=0, fill=3):
if name is None:
count = Default.get_counter(name='name') + offset
prefix = Default.user
if prefix is None or count is None:
Console.error("Prefix and Count could not be retrieved correctly.", traceflag=False)
return
name = prefix + "-" + str(count).zfill(fill)
return name
def _refresh_cloud(cloud):
try:
msg = "Refresh VMs for cloud {:}.".format(cloud)
if Vm.refresh(cloud=cloud):
Console.ok("{:} OK.".format(msg))
else:
Console.error("{:} failed".format(msg), traceflag=False)
except Exception as e:
Console.error("Problem running VM refresh", traceflag=False)
def _get_vm_names():
vm_list = cm.find(kind="vm")
vms = [vm["name"] for vm in vm_list]
names = pattern = arguments["NAMES"]
if pattern is not None:
if "*" in pattern:
names = search(vms, pattern)
else:
names = Parameter.expand(names)
if names == ['last'] or names is None:
names == [Default.vm]
return vm_list, names
cloud = arguments["--cloud"] or Default.cloud
config = ConfigDict("cloudmesh.yaml")
active_clouds = config["cloudmesh"]["active"]
def _refresh(cloud):
all = arguments["all"] or None
if all is None:
_refresh_cloud(cloud)
else:
for cloud in active_clouds:
_refresh_cloud(cloud)
arg = dotdict(arguments)
arg.cloud = arguments["--cloud"] or Default.cloud
arg.image = arguments["--image"] or Default.get(name="image", category=arg.cloud)
arg.flavor = arguments["--flavor"] or Default.get(name="flavor", category=arg.cloud)
arg.group = arguments["--group"] or Default.group
arg.secgroup = arguments["--secgroup"] or Default.secgroup
arg.key = arguments["--key"] or Default.key
arg.dryrun = arguments["--dryrun"]
arg.name = arguments["--name"]
arg.format = arguments["--format"] or 'table'
arg.refresh = Default.refresh or arguments["--refresh"]
arg.count = int(arguments["--n"] or 1)
arg.dryrun = arguments["--dryrun"]
arg.verbose = not arguments["--quiet"]
#
# in many cases use NAMES
# if arg.NAMES is not None:
# arg.names = Parameter.expand(arg.NAMES) # gvonlasz[001-002] gives ["gvonlasz-001", "gvonlasz-002"]
# else:
# arg.names = None
#
if arguments["boot"]:
arg.username = arguments["--username"] or Image.guess_username(arg.image)
is_name_provided = arg.name is not None
arg.user = Default.user
for index in range(0, arg.count):
vm_details = dotdict({
"cloud": arg.cloud,
"name": get_vm_name(arg.name, index),
"image": arg.image,
"flavor": arg.flavor,
"key": arg.key,
"secgroup": arg.secgroup,
"group": arg.group,
"username": arg.username,
"user": arg.user
})
# correct the username
vm_details.username = Image.guess_username_from_category(
vm_details.cloud,
vm_details.image,
username=arg.username)
try:
if arg.dryrun:
print(Printer.attribute(vm_details, output=arg.format))
msg = "dryrun info. OK."
Console.ok(msg)
else:
vm_id = Vm.boot(**vm_details)
if vm_id is None:
msg = "info. failed."
Console.error(msg, traceflag=False)
return ""
# set name and counter in defaults
Default.set_vm(value=vm_details.name)
if is_name_provided is False:
Default.incr_counter("name")
# Add to group
if vm_id is not None:
Group.add(name=vm_details.group,
species="vm",
member=vm_details.name,
category=vm_details.cloud)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Console.error("Problem booting instance {name}".format(**vm_details), traceflag=False)
elif arguments["username"]:
arg.username = arguments["--username"] or Image.guess_username(arg.image)
cloud = arg.cloud
username = arg.USERNAME
if arg.NAMES is None:
names = [Default.vm]
else:
names = Parameter.expand(arg.NAMES)
if len(names) == 0:
return
for name in names:
arg.name = name
Console.ok("Set username for {cloud}:{name} to {USERNAME}".format(**arg))
Vm.set_login_user(name=name, cloud=cloud, username=username)
elif arguments["default"]:
try:
count = Default.get_counter()
prefix = Username()
if prefix is None or count is None:
Console.error("Prefix and Count could not be retrieved correctly.", traceflag=False)
return
vm_name = prefix + "-" + str(count).zfill(3)
arg = {
"name": vm_name,
"cloud": arguments["--cloud"] or Default.cloud
}
for attribute in ["image", "flavor"]:
arg[attribute] = Default.get(name=attribute, category=cloud)
for attribute in ["key", "group", "secgroup"]:
arg[attribute] = Default.get(name=attribute, category='general')
output = arguments["--format"] or "table"
print(Printer.attribute(arg, output=output))
msg = "info. OK."
Console.ok(msg)
ValueError("default command not implemented properly. Upon "
"first install the defaults should be read from yaml.")
except Exception as e:
# Error.traceback(e)
Console.error("Problem listing defaults", traceflag=False)
elif arguments["ping"]:
try:
if arguments["NAME"] is None and arguments["N"] is None:
name = arguments["NAME"] or Default.vm
n = arguments["N"] or 1
elif arguments["NAME"].isdigit():
n = arguments["NAME"]
name = Default.vm
else:
name = arguments["NAME"] or Default.vm
n = arguments["N"] or 1
print("Ping:", name, str(n))
vm = dotdict(Vm.list(name=name, category=cloud, output="dict")["dict"])
ip = vm.floating_ip
result = Shell.ping(host=ip, count=n)
print(result)
except Exception as e:
Console.error(e.message, traceflag=False)
elif arguments["console"]:
try:
name = arguments["NAME"] or Default.vm
vm = dotdict(Vm.list(name=name, category=cloud, output="dict")["dict"])
cloud_provider = CloudProvider(cloud).provider
vm_list = cloud_provider.list_console(vm.uuid)
print(vm_list)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
# Error.traceback(e)
Console.error("Problem retrieving status of the VM", traceflag=False)
elif arguments["status"]:
try:
cloud_provider = CloudProvider(cloud).provider
vm_list = cloud_provider.list_vm(cloud)
vms = [vm_list[i]["name"] for i in vm_list ]
print ("V", vms)
pattern = arguments["NAMES"]
if pattern is not None:
if "*" in pattern:
print ("serach")
names = search(vms, pattern)
else:
names = Parameter.expand()
for i in vm_list:
if vm_list[i]["name"] in names:
print("{} {}".format(vm_list[i]["status"], vm_list[i]["name"]))
else:
print("{} {}".format(vm_list[0]["status"], vm_list[0]["name"]))
except Exception as e:
# Error.traceback(e)
Console.error("Problem retrieving status of the VM", traceflag=True)
elif arguments["wait"]:
interval = arguments["--interval"] or 5
try:
cloud_provider = CloudProvider(cloud).provider
for i in range(1,10):
vm_list = cloud_provider.list_vm(cloud)
time.sleep(float(1))
d = {}
for id in vm_list:
vm = vm_list[id]
d[vm["name"]] = vm["status"]
print (d)
print("{} {}".format(vm_list[0]["status"], vm_list[0]["name"]))
if vm_list[0]["status"] in ['ACTIVE']:
return
except Exception as e:
# Error.traceback(e)
Console.error("Problem retrieving status of the VM", traceflag=True)
elif arguments["info"]:
try:
cloud_provider = CloudProvider(cloud).provider
vms = cloud_provider.list_vm(cloud)
vm = vms[0]
output_format = arguments["--format"] or "table"
print(Printer.attribute(vm, output=output_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
# Error.traceback(e)
Console.error("Problem retrieving status of the VM", traceflag=False)
elif arguments["check"]:
test = {}
try:
names = Parameter.expand(arguments["NAME"])
id = 0
for name in names:
print("Not implemented: {}".format(name))
# TODO: check the status of the vms
status = "active"
# TODO: check if they have a floating ip
# TODO: get ip
floating_ip = "127.0.0.1"
ip = True
# ping
# TODO: ping the machine with the shell command
ping = True
# check if one can login and run a command
check = False
try:
r = Shell.execute("uname", "-a")
# do a real check
check = True
except:
check = False
test[name] = {
"id": id,
"name": name,
"status": status,
"ip": ip,
"ping": ping,
"login": check
}
id += 1
pprint(test)
print(Printer.write(test,
order=["id",
"name",
"status",
"ip",
"ping",
"login"],
output="table",
sort_keys=True))
msg = "not yet implemented. failed."
Console.error(msg, traceflag=False)
except Exception as e:
# Error.traceback(e)
Console.error("Problem retrieving status of the VM", traceflag=False)
elif arguments["start"]:
try:
servers = Parameter.expand(arguments["NAMES"])
# If names not provided, take the last vm from DB.
if len(servers) == 0:
last_vm = Default.vm
if last_vm is None:
Console.error("No VM records in database. Please run vm refresh.", traceflag=False)
return ""
name = last_vm["name"]
# print(name)
servers = list()
servers.append(name)
group = arguments["--group"]
force = arguments["--force"]
# if default cloud not set, return error
if not cloud:
Console.error("Default cloud not set.", traceflag=False)
return ""
Vm.start(cloud=cloud, servers=servers)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
# Error.traceback(e)
Console.error("Problem starting instances", traceflag=False)
elif arguments["stop"]:
try:
servers = Parameter.expand(arguments["NAMES"])
# If names not provided, take the last vm from DB.
if servers is None or len(servers) == 0:
last_vm = Default.vm
if last_vm is None:
Console.error("No VM records in database. Please run vm refresh.", traceflag=False)
return ""
name = last_vm["name"]
# print(name)
servers = list()
servers.append(name)
group = arguments["--group"]
force = arguments["--force"]
# if default cloud not set, return error
if not cloud:
Console.error("Default cloud not set.", traceflag=False)
return ""
Vm.stop(cloud=cloud, servers=servers)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
# Error.traceback(e)
Console.error("Problem stopping instances", traceflag=False)
elif arguments["refresh"]:
_refresh(cloud)
elif arguments["delete"]:
dryrun = arguments["--dryrun"]
group = arguments["--group"]
force = not arguments["--keep"]
cloud = arguments["--cloud"]
vms, servers = _get_vm_names()
if servers in [None, []]:
Console.error("No vms found.", traceflag=False)
return ""
for server in servers:
if dryrun:
Console.ok("Dryrun: delete {}".format(server))
else:
Vm.delete(servers=[server], force=force)
return ""
elif arguments["ip"] and arguments["assign"]:
if arguments["NAMES"] is None:
names = [Default.vm]
else:
names = Parameter.expand(arguments["NAMES"])
for name in names:
# ip = Network.get_floatingip(....)
vm = dotdict(Vm.list(name=name, category=cloud, output="dict")["dict"])
if vm.floating_ip is None:
Console.ok("Assign IP to {}".format(name))
try:
floating_ip = Network.find_assign_floating_ip(cloudname=cloud,
instance_id=name)
Vm.refresh(cloud=cloud)
if floating_ip is not None:
print(
"Floating IP assigned to {:} is {:}".format(
name, floating_ip))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Console.error("Problem assigning floating ips.", traceflag=False)
else:
Console.error("VM {} already has a floating ip: {}".format(name, vm.floating_ip), traceflag=False)
elif arguments["ip"] and arguments["inventory"]:
vms, names = _get_vm_names()
if names in [None, []]:
if str(Default.vm) in ['None', None]:
Console.error("The default vm is not set.", traceflag=False)
return ""
else:
names = [Default.vm]
header = arguments["--header"] or "[servers]"
filename = arguments["--file"] or "inventory.txt"
try:
vm_ips = []
for vm in vms:
if vm["name"] in names:
print (vm["name"])
vm_ips.append(vm["floating_ip"])
result = header + "\n"
result += '\n'.join(vm_ips)
Console.ok("Creating inventory file: {}".format(filename))
Console.ok(result)
with open(filename, 'w') as f:
f.write(result)
except Exception as e:
Console.error("Problem getting ip addresses for instance", traceflag=True)
elif arguments["ip"] and arguments["show"]:
if arguments["NAMES"] is None:
if str(Default.vm) in ['None', None]:
Console.error("The default vm is not set.", traceflag=False)
return ""
else:
names = [Default.vm]
else:
names = Parameter.expand(arguments["NAMES"])
group = arguments["--group"]
output_format = arguments["--format"] or "table"
refresh = arguments["--refresh"]
try:
ips = Ip.list(cloud=arg.cloud, output=output_format, names=names)
print(ips)
except Exception as e:
Console.error("Problem getting ip addresses for instance", traceflag=False)
elif arguments["ssh"]:
def _print(msg):
if arg.verbose:
Console.msg(msg)
chameleon = "chameleon" in ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["clouds"][arg.cloud][
"cm_host"]
if chameleon:
arg.username = "******"
elif arg.cloud == "azure":
arg.username = ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["clouds"]["azure"]["default"]["username"]
else:
if arg.username is None:
Console.error("Could not guess the username of the vm", traceflag=False)
return
arg.username = arguments["--username"] or Image.guess_username(arg.image)
arg.command = arguments["--command"]
data = dotdict({
'name': arguments["NAME"] or Default.vm,
'username': arg.username,
'cloud': arg.cloud,
'command': arg.command
})
_print("login {cloud}:{username}@{name}".format(**data))
vm = Vm.get(data.name, category=data.cloud)
Vm.set_login_user(name=data.name, cloud=data.cloud, username=data.username)
data.floating_ip = vm.floating_ip
data.key = arguments["--key"] or Default.key
_print(Printer.attribute(data))
'''
if vm.username is None:
user_from_db = Vm.get_login_user(vm.name, vm.cloud)
user_suggest = user_from_db or Default.user
username = input("Username (Default: {}):".format(user_suggest)) or user_suggest
Vm.set_login_user(name=data.name, cloud=cloud, username=data.username)
'''
ip = arguments["--ip"]
commands = arguments["--command"]
ip_addresses = []
cloud_provider = CloudProvider(cloud).provider
ip_addr = cloud_provider.get_ips(vm.name)
ipaddr_dict = Vm.construct_ip_dict(ip_addr, cloud)
for entry in ipaddr_dict:
ip_addresses.append(ipaddr_dict[entry]["addr"])
if len(ip_addresses) > 0:
if ip is not None:
if ip not in ip_addresses:
Console.error("IP Address specified does not match with the host.", traceflag=False)
return ""
else:
_print("Determining IP Address to use with a ping test.")
# This part assumes that the ping is allowed to the machine.
for ipadd in ip_addresses:
_print("Checking {:}...".format(ipadd))
try:
# Evading ping test, as ping is not enabled for VMs on Azure cloud
# socket.gethostbyaddr(ipadd)
# ip will be set if above command is successful.
ip = ipadd
except socket.herror:
_print("Cannot reach {:}.".format(ipadd))
if ip is None:
_print("Unable to connect to the machine")
return ""
else:
_print("IP to be used is: {:}".format(ip))
#
# TODO: is this correctly implemented
#
if not cloud == 'azure':
SecGroup.enable_ssh(cloud=cloud)
if arg.verbose:
Console.info("Connecting to Instance at IP:" + format(ip))
# Constructing the ssh command to connect to the machine.
sshcommand = "ssh"
if arg.key is not None:
sshcommand += " -i {:}".format(arg.key)
sshcommand += " -o StrictHostKeyChecking=no"
sshcommand += " {:}@{:}".format(data.username, ip)
if commands is not None:
sshcommand += " \"{:}\"".format(commands)
# print(sshcommand)
os.system(sshcommand)
else:
Console.error("No Public IPs found for the instance", traceflag=False)
elif arguments["list"]:
# groups = Group.list(output="dict")
arg = dotdict(arguments)
arg.names = arguments["NAMES"]
arg.group = arguments["--group"]
if arg.group is None:
arg.group = []
else:
arg.group = Parameter.expand(arguments["--group"])
arg.refresh = arguments["--refresh"] or Default.refresh
if arg.NAMES is not None:
arg.names = Parameter.expand(arguments["NAMES"])
else:
arg.names = ["all"]
_format = arguments["--format"] or "table"
if arguments["--active"]:
clouds = active_clouds
else:
if arguments["--cloud"]:
clouds = Parameter.expand(arguments["--cloud"])
else:
clouds = [Default.cloud]
try:
d = ConfigDict("cloudmesh.yaml")
for cloud in clouds:
if arg.refresh:
_refresh(cloud)
Console.ok("Listing VMs on Cloud: {:}".format(cloud))
vms = Vm.list(category=cloud, output="raw")
# print ("XXX", type(vms), vms)
if vms is None:
break
result = []
if "all" in arg.names:
if result is None:
result = []
else:
result = vms
elif arg.group is not None and len(arg.group) > 0:
for vm in vms:
if vm["group"] in arg.group:
result.append(vm)
elif arg.names is not None and len(arg.names) > 0:
for vm in vms:
if vm["name"] in arg.names:
result.append(vm)
if len(result) > 0:
# print(result)
(order, header) = CloudProvider(cloud).get_attributes("vm")
print(Printer.write(result,
order=order,
output=_format)
)
else:
Console.error("No data found with requested parameters.", traceflag=False)
except Exception as e:
# Error.traceback(e)
Console.error("Problem listing all instances", traceflag=False)
elif arguments["rename"]:
try:
oldnames = Parameter.expand(arguments["OLDNAMES"])
newnames = Parameter.expand(arguments["NEWNAMES"])
force = arguments["--force"]
if oldnames is None or newnames is None:
Console.error("Wrong VMs specified for rename", traceflag=False)
elif len(oldnames) != len(newnames):
Console.error("The number of VMs to be renamed is wrong",
traceflat=False)
else:
for i in range(0, len(oldnames)):
oldname = oldnames[i]
newname = newnames[i]
if arguments["--dryrun"]:
Console.ok("Rename {} to {}".format(oldname, newname))
else:
Vm.rename(cloud=cloud,
oldname=oldname,
newname=newname,
force=force
)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
# Error.traceback(e)
Console.error("Problem deleting instances", traceflag=False)
return ""
