def do_terminal(self, args, arguments):
"""
::
Usage:
terminal [--os=OS]
[--command=COMMAND]
[--shell=SHELL]
[--interactive=INTERACTIVE]
[--window=WINDOW]
Starts a docker container in interactive mode in a new terminal
and executes the command in it.
Arguments:
--command=COMMAND the command
--os=OS the os [default: cloudmesh/book:latest]
--shell=SHELL [default: /bin/bash]
--window=WINDOW [default: True]
--interactive=INTERACTIVE [default: True]
Options:
-f specify the file
Description:
terminal --os="cloudmesh/book:1.7" --command=ls
"""
# m = Manager()
map_parameters(arguments, 'os', 'command', 'interactive', 'shell',
'window')
arguments.cwd = os.getcwd()
if arguments.command is None:
arguments.command = ""
else:
arguments.command += ";"
VERBOSE.print(arguments, label='arguments', verbose=1)
if arguments.os is None:
Shell.terminal(command=arguments.command)
else:
if arguments.interactive.lower() in ['true', 'on']:
arguments.interactive = "-it"
else:
arguments.interactive = ""
arguments.shell = ""
VERBOSE.print(arguments, label='arguments', verbose=1)
command = "cd {cwd}; docker run -v {cwd}:{cwd} -w {cwd} --rm {interactive} {os} {command}{shell}".format(
**arguments)
print(command)
Shell.terminal(command=command)
return ""
def __init__(self,
name=None,
configuration="~/.cloudmesh/cloudmesh4.yaml"):
"""
Initializes the provider. The default parameters are read from the configutation
file that is defined in yaml format.
:param name: The name of the provider as defined in the yaml file
:param configuration: The location of the yaml configuration file
"""
conf = Config(configuration)["cloudmesh"]
self.user = conf["profile"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
super().__init__(name, conf)
VERBOSE.print(cred, verbose=8)
if self.cloudtype in Provider.ProviderMapper:
self.driver = get_driver(Provider.ProviderMapper[self.cloudtype])
if self.cloudtype == 'openstack':
if cred["OS_PASSWORD"] == 'TBD':
Console.error("The password TBD is not allowed")
self.cloudman = self.driver(
cred["OS_USERNAME"],
cred["OS_PASSWORD"],
ex_force_auth_url=cred['OS_AUTH_URL'],
ex_force_auth_version='2.0_password',
ex_tenant_name=cred['OS_TENANT_NAME'])
elif self.cloudtype == 'azure_asm':
self.cloudman = self.driver(
subscription_id=cred['AZURE_SUBSCRIPTION_ID'],
key_file=path_expand(cred['AZURE_KEY_FILE']))
elif self.cloudtype == 'aws':
self.cloudman = self.driver(cred["EC2_ACCESS_ID"],
cred["EC2_SECRET_KEY"],
region=cred["EC2_REGION"])
if self.cloudtype == 'google':
self.cloudman = self.driver(
cred["client_email"],
cred[
"path_to_json_file"], # should be placed in .cloudmesh
project=cred["project"])
else:
print("Specified provider not available")
self.cloudman = False
self.default_image = None
self.default_size = None
self.public_key_path = conf["profile"]["publickey"]
def __init__(self, config_path='~/.cloudmesh/cloudmesh4.yaml',
encrypted=False):
"""
Initialize the Config class.
:param config_path: A local file path to cloudmesh yaml config
with a root element `cloudmesh`. Default: `~/.cloudmesh/cloudmesh4.yaml`
"""
self.__dict__ = self.__shared_state
if "data" not in self.__dict__:
VERBOSE.print("Load config", verbose=9)
self.config_path = Path(path_expand(config_path)).resolve()
self.config_folder = dirname(self.config_path)
if not exists(self.config_folder):
mkdir(self.config_folder)
if not isfile(self.config_path):
source = Path(join(dirname(realpath(__file__)),
"../../etc/cloudmesh4.yaml"))
copyfile(source.resolve(), self.config_path)
#with open(self.config_path, "r") as stream:
# # self.data = yaml.load(stream, Loader=yaml.FullLoader)
# self.data = yaml.load(stream, Loader=yaml.SafeLoader)
with open(self.config_path, "r") as stream:
content = stream.read()
content = path_expand(content)
content = self.spec_replace(content)
self.data = yaml.load(content, Loader=yaml.SafeLoader)
# self.data is loaded as nested OrderedDict, can not use set or get methods directly
if self.data is None:
raise EnvironmentError(
"Failed to load configuration file cloudmesh4.yaml, please check the path and file locally")
#
# populate default variables
#
self.variable_database = Variables(filename="~/.cloudmesh/var-data")
self.set_debug_defaults()
default = self.default()
for name in self.default():
if name not in self.variable_database:
self.variable_database[name] = default[name]
if "cloud" in default:
self.cloud = default["cloud"]
else:
self.cloud = None
def __init__(self, filename, secret):
self.data = dotdict({
'file': filename,
'secret': secret,
'pem': path_expand('~/.ssh/id_rsa.pem'),
'key': path_expand('~/.ssh/id_rsa')
})
VERBOSE.print(self.data, verbose=9)
if not os.path.exists(self.data["pem"]):
self.pem_create()
def __init__(self,
name=None,
configuration="~/.cloudmesh/.cloudmesh4.yaml"):
VERBOSE.print(f"Init Docker {name}", verbose=9)
self.config = Config()
conf = Config(configuration)["cloudmesh"]
self.user = conf["profile"]
self.spec = conf["cloud"][name]
self.cloud = name
cred = self.spec["credentials"]
self.cloudtype = self.spec["cm"]["kind"]
def test_04_list_flavors(self):
HEADING()
flavors = self.p.flavors()
# pprint (flavors)
VERBOSE.print(flavors, verbose=9)
print(
Printer.flatwrite(flavors,
sort_keys=["name", "vcpus", "disk"],
order=["name", "vcpus", "ram", "disk"],
header=["Name", "VCPUS", "RAM", "Disk"]))
def test_03_list_images(self):
HEADING()
images = self.p.images()
VERBOSE.print(images, verbose=9)
print(
Printer.flatwrite(
images,
sort_keys=["name", "extra.minDisk"],
order=["name", "extra.minDisk", "updated", "driver"],
header=["Name", "MinDisk", "Updated", "Driver"]))
def test_04_list_vm(self):
HEADING()
vms = self.p.list()
# pprint (vms)
VERBOSE.print(vms, verbose=9)
print(
Printer.flatwrite(vms,
sort_keys=("name"),
order=[
"name", "state", "extra.task_state",
"extra.vm_state", "extra.userId",
"extra.key_name", "private_ips", "public_ips"
],
header=[
"Name", "State", "Task state", "VM state",
"User Id", "SSHKey", "Private ips",
"Public ips"
]))
def do_config(self, args, arguments):
"""
::
Usage:
config -h | --help
config encrypt [SOURCE]
config decrypt [SOURCE]
config edit [SOURCE]
config set ATTRIBUTE=VALUE
config set ATTRIBUTE
config ssh keygen
config ssh verify
config ssh check
config ssh pem
Arguments:
SOURCE the file to encrypted or decrypted.
an .enc is added to the filename or removed form it
dependent of if you encrypt or decrypt
ATTRIBUTE=VALUE sets the attribute with . notation in the
configuration file.
ATTRIBUTE reads the attribute from the terminal and sets it
in the configuration file
If the attribute is a password, * is written instead
of the character included
Options:
--name=KEYNAME The name of a key
Description:
config check
checks if the ssh key ~/.ssh/id_rsa has a password. Verifies it
through entering the passphrase
Key generation
Keys must be generated with
ssh-keygen -t rsa -m pem
openssl rsa -in ~/.ssh/id_rsa -out ~/.ssh/id_rsa.pem
or
cms config ssh keygen
Key validity can be checked with
cms config check
The key password can be verified with
cms config verify
ssh-add
cms config encrypt ~/.cloudmesh/cloudmesh4.yaml
cms config decrypt ~/.cloudmesh/cloudmesh4.yaml
config set ATTRIBUTE=VALUE
config set profile.name=Gregor
"""
# d = Config() #~/.cloudmesh/cloudmesh4.yaml
# d = Config(encryted=True) # ~/.cloudmesh/cloudmesh4.yaml.enc
arguments.SOURCE = arguments.SOURCE or \
path_expand("~/.cloudmesh/cloudmesh4.yaml")
arguments.DESTINATION = arguments.SOURCE + ".enc"
VERBOSE.print(arguments, verbose=9)
e = EncryptFile(arguments.SOURCE, arguments.DESTINATION)
if arguments.encrypt:
e.encrypt()
Console.ok("{SOURCE} --> {DESTINATION}".format(**arguments))
Console.ok("file encrypted")
return ""
elif arguments.decrypt:
# if the file is existed
if not os.path.exists(arguments.DESTINATION):
Console.error(
"encrypted file {DESTINATION} does not exist".format(
**arguments))
e.decrypt(arguments.SOURCE, arguments.DESTINATION)
Console.ok("{DESTINATION} --> {SOURCE}".format(**arguments))
Console.ok("file decrypted")
return ""
elif arguments.ssh and arguments.verify:
e.pem_verify()
elif arguments.ssh and arguments.check:
key = "~/.ssh/id_rsa"
r = e.check_key(key)
if r:
Console.ok(f"Key {key} is valid")
# does not work as it does not change it to pem format
# e.check_passphrase()
elif arguments.ssh and arguments.pem:
r = e.pem_create()
elif arguments.set:
Console.error("not implemented")
raise NotImplementedError
elif arguments.ssh and arguments.keygen:
e.ssh_keygen()
return ""
def do_vm(self, args, arguments):
"""
::
Usage:
vm start [--cloud=CLOUD]
[--name=NAME]
vm stop [--cloud=CLOUD]
[--name=NAME]
vm delete [--cloud=CLOUD]
[--name=NAME]
vm list [--cloud=CLOUDS]
vm boot [--name=NAME]
[--cloud=CLOUD]
[--image=IMAGE]
[--flavor=FLAVOR]
vm ssh [--cloud=CLOUD]
[--name=NAME]
[--command=COMMAND]
vm info [--cloud=CLOUD]
[--format=FORMAT]
vm resize [NAMES] [--size=SIZE]
vm images [--cloud=CLOUD]
vm flavors [--cloud=CLOUD]
Arguments:
NAMES server name. By default it is set to the name of last vm from database.
NAME server name.
Options:
--ip=IP give the public ip of the server
--cloud=CLOUD give a cloud to work on, if not given, selected
or default cloud will be used
--flavor=FLAVOR give the name or id of the flavor
--image=IMAGE give the name or id of the image
--command=COMMAND specify the commands to be executed
Description:
commands used to boot, start or delete servers of a cloud
vm boot [options...]
Boots servers on a cloud, user may specify flavor, image
.etc, otherwise default values will be used, see how to set
default values of a cloud: cloud help
vm start [options...]
Starts a suspended or stopped vm instance.
vm stop [options...]
Stops a vm instance .
vm delete [options...]
Delete servers of a cloud, user may delete a server by its
name or id, delete servers of a group or servers of a cloud,
give prefix and/or range to find servers by their names.
Or user may specify more options to narrow the search
vm ssh [options...]
login to a server or execute commands on it
vm list [options...]
same as command "list vm", please refer to it
vm status [options...]
Retrieves status of the VM requested
vm refresh [--cloud=CLOUDS]
this command refreshes the data for virtual machines,
images and flavors for the specified clouds.
Tip:
give the VM name, but in a hostlist style, which is very
convenient when you need a range of VMs e.g. sample[1-3]
=> ['sample1', 'sample2', 'sample3']
sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18']
Quoting commands:
cm vm login gvonlasz-004 --command=\"uname -a\"
"""
def map_parameters(arguments, *args):
for arg in args:
flag = "--" + arg
if flag in arguments:
arguments[arg] = arguments[flag]
else:
arguments[arg] = None
def get_cloud_and_names(label, arguments):
names = []
clouds = []
if arguments["--cloud"]:
clouds = get_clouds(arguments, variables)
else:
clouds = get_clouds(arguments, variables)
names = get_names(arguments, variables)
return clouds, names
def get_cloud_and_names_commands(label, arguments):
names = []
clouds = []
commands = []
if arguments["--cloud"]:
clouds = get_clouds(arguments, variables)
else:
clouds = get_clouds(arguments, variables)
names = get_names(arguments, variables)
commands = get_commands(arguments, variables)
return clouds, names, commands
def get_clouds(arguments, variables):
clouds = arguments["cloud"] or arguments["--cloud"]
if "active" == clouds:
active = Active()
clouds = active.clouds()
else:
clouds = clouds
if (clouds is None) or (clouds == ""):
Console.error("you need to specify a cloud")
return None
return clouds
def get_names(arguments, variables):
names = arguments["NAME"] or arguments["NAMES"] or arguments["--name"] or variables["vm"]
if names is None:
Console.error("you need to specify a vm")
return None
else:
return names
def get_image(arguments, variables):
image = arguments["image"] or arguments["--image"]
if image is None:
Console.error("you need to specify an image")
return None
else:
return image
def get_command(arguments, variables):
command = arguments["command"] or arguments["--command"]
if command is None:
Console.error("you need to specify a command")
return None
else:
return command
def get_flavor(arguments, variables):
flavor = arguments["flavor"] or arguments["--flavor"]
if flavor is None:
Console.error("you need to specify a flavor")
return None
else:
return flavor
def get_commands(label, arguments):
names = []
if label in ["delete", "stop", "start"]:
clouds = get_clouds(arguments, variables)
names = get_names(arguments, variables)
return clouds, names
elif label in ["list", "flavors", "images"]:
clouds = get_clouds(arguments, variables)
return clouds
elif "boot" == label:
clouds = get_clouds(arguments, variables)
names = get_names(arguments, variables)
image = get_image(arguments, variables)
flavor = get_flavor(arguments, variables)
return clouds, names, image, flavor
elif "ssh" == label:
clouds = get_clouds(arguments, variables)
names = get_names(arguments, variables)
command = get_command(arguments, variables)
return clouds, names, command
map_parameters(arguments,
'cloud',
'command',
'flavor',
'format',
'image',
'ip',
'name',
'NAME')
VERBOSE.print(arguments, verbose=9)
variables = Variables()
if arguments.images:
clouds = get_commands("images", arguments)
if clouds is None:
return ""
else:
p = Provider(clouds)
images = p.p.images()
print(Printer.flatwrite(images,
sort_keys=("name"),
order=["name", "id", "driver"],
header=["Name", "Id", "Driver"])
)
elif arguments.flavors:
clouds = get_commands("flavors", arguments)
if clouds is None:
return ""
else:
p = Provider(clouds)
flavors = p.p.flavors()
print(Printer.flatwrite(flavors,
sort_keys=("name", "disk"),
order=["name", "id", "ram", "disk"],
header=["Name", "Id", "RAM", "Disk"])
)
elif arguments.boot:
print("Creating a new vm")
clouds, names, image, flavor = get_commands("boot", arguments)
if clouds is None or names is None or image is None or flavor is None:
return ""
else:
p = Provider(clouds)
node = p.p.create(name=names, size=flavor, image=image)
print(Printer.flatwrite(node,
sort_keys=("name"),
order=["name", "state", "public_ips", "private_ips", "size", "image"],
header=["Name", "State", "Public IP", "Private IP", "Size", "Image"])
)
elif arguments.start:
print("Starting the requested vm")
clouds, names = get_commands("start", arguments)
if clouds is None or names is None:
return ""
else:
p = Provider(clouds)
node = p.p.start(name=names)
elif arguments.stop:
print("Stopping the requested vm")
clouds, names = get_commands("stop", arguments)
if clouds is None or names is None:
return ""
else:
p = Provider(clouds)
node = p.p.stop(name=names)
elif arguments.delete:
print("Delete the specified VM")
clouds, names = get_commands("delete", arguments)
if clouds is None or names is None:
return ""
else:
p = Provider(clouds)
p.p.destroy(name=names)
return ""
elif arguments.list:
print("list the vms in the cloud")
clouds = get_commands("list", arguments)
if clouds is None:
return ""
else:
p = Provider(clouds)
vms = p.p.list()
print(Printer.flatwrite(vms,
sort_keys=("name"),
order=["name", "state", "public_ips", "private_ips", "size", "image"],
header=["Name", "State", "Public IP", "Private IP", "Size", "Image"])
)
elif arguments.resize:
"""
vm resize [NAMES] [--size=SIZE]
"""
pass
elif arguments.ssh:
print("ssh into the vm and execute command")
clouds, names, command = get_commands("ssh", arguments)
if clouds is None or names is None or command is None:
return ""
else:
p = Provider(clouds)
p.p.ssh(name=names, command=command)
def do_batch(self, args, arguments):
"""
::
Usage:
batch job create
--name=NAME
--cluster=CLUSTER
--script=SCRIPT
--executable=EXECUTABLE
--destination=DESTINATION
--source=SOURCE
[--companion-file=COMPANION_FILE]
[--outfile-name=OUTPUT_FILE_NAME]
[--suffix=SUFFIX]
[--overwrite]
batch job run [--name=NAMES] [--format=FORMAT]
batch job fetch [--name=NAMES]
batch job remove [--name=NAMES]
batch job clean [--name=NAMES]
batch job set [--name=NAMES] PARAMETER=VALUE
batch job list [--name=NAMES] [--depth=DEPTH]
batch cluster test [--cluster=CLUSTERS]
batch cluster list [--cluster=CLUSTERS] [--depth=DEPTH]
batch cluster remove [--cluster=CLUSTERS]
batch cluster set [--cluster=CLUSTERS] PARAMETER=VALUE
Arguments:
FILE a file name
INPUT_TYPE tbd
Options:
-f specify the file
--depth=DEPTH [default: 1]
--format=FORMAT [default: table]
Description:
This command allows to submit batch jobs to queuing systems hosted
in an HBC center as a service directly form your commandline.
We assume that a number of experiments are conducted with possibly
running the script multiple times. Each experiment will save the
batch script in its own folder.
The output of the script can be saved in a destination folder. A virtual
directory is used to coordinate all saved files.
The files can be located due to the use of the virtual directory on
multiple different data or file services
Authentication to the Batch systems is done viw the underlaying HPC
center authentication. We assume that the user has an account to
submit on these systems.
(SSH, 2 factor, XSEDE-account) TBD.
Experiments:
experiments are jobs that can be run multiple times and create input
and output file sin them
cloudmesh:
experiment:
job:
name: {cloudmesh.profile.user.name}-01
directory: ~/experiment/{experiment.job.name}
output: {cloudmesh.experiment.job.name}/output
input: ~/experiment/{experiment.job.name}/input
script: script.sh
source ,,,
destination: {cloudmesh.experiment.job.directory}
I do not know what companion file is
Examples:
batch job run [--name=NAMES] [--format=FORMAT]
runs jobs with the given names
LOTS OF DOCUMENTATION MISSING HERE
[--companion-file=COMPANION_FILE]
[--outfile-name=OUTPUT_FILE_NAME]
[--suffix=SUFFIX] [--overwrite]
"""
#
# create slurm manager so it can be used in all commands
#
slurm_manager = SlurmCluster() # debug=arguments["--debug"])
arguments["--cloud"] = "test"
arguments["NAME"] = "fix"
map_parameters(arguments,
"cloud",
"name",
"cluster",
"script",
"type",
"destination",
"source",
"format")
# if not arguments.create
# find cluster name from Variables()
# if no cluster is defined look it up in yaml in batch default:
# if not defined there fail
# clusters = Parameter.expand(arguments.cluster)
# name = Parameters.expand[argumnets.name)
# this will return an array of clusters and names of jobs and all cluster
# job or clusterc commands will be executed on them
# see the vm
#
# if active: False in the yaml file for the cluster this cluster is not used and scipped.
VERBOSE.print(arguments, verbose=9)
variables = Variables()
# do not use print but use ,Console.msg(), Console.error(), Console.ok()
if arguments.tester:
print("running ... ")
slurm_manager.tester()
elif arguments.run and arguments.job:
# config = Config()["cloudmesh.batch"]
names = Parameter.expand(arguments.name)
# clouds, names = Arguments.get_cloud_and_names("refresh", arguments,
# variables)
data = []
for name in names:
entry = SlurmCluster.job_specification()
data.append(entry)
'''
data = {
"cm": {
"cloud": "karst_debug",
"kind": "batch-job",
"name": "job012",
},
"batch": {
"source": "~/.cloudmesh/batch/dir",
"destination": "~/.cloudmesh/dir/",
"status": "running"
}
}'''
try:
raise NotImplementedError
except Exception as e:
Console.error("Haha", traceflag=True)
pprint(data)
print(Printer.flatwrite(
data,
order=["cm.name", "cm.kind", "batch.status"],
header=["Name", "Kind", "Status"],
output=arguments.format)
)
return ""
elif arguments.job and arguments.create:
# if not arguments.name:
# raise ValueError
# assert input_type in ['params', 'params+file'], "Input type can be either params or params+file"
# if input_type == 'params+file':
# assert arguments.get("--argfile-path") is not None, "Input type is params+file but the input \
# filename is not specified"
job_name = arguments.name
cluster_name = arguments.cluster
script_path = Path(arguments.script)
if not script_path.exists():
raise FileNotFoundError
executable_path = Path(arguments['--executable'])
if not executable_path.exists():
raise FileNotFoundError
destination = Path(arguments.destination)
if not destination.is_absolute():
Console.error("destination path must be absolute",
traceflag=True)
raise FileNotFoundError
source = Path(arguments.source)
if not source.exists():
raise FileNotFoundError
if arguments.experiment is None:
experiment_name = 'job' + self.suffix_generator()
else:
experiment_name = arguments.experiment + self.suffix_generator()
# overwrite = False if type(arguments.get("--overwrite")) is None else arguments.get("--overwrite")
if arguments.get("--companion-file") is None:
companion_file = Path()
else:
companion_file = Path(arguments.get("--companion-file"))
slurm_manager.create(job_name,
cluster_name,
script_path,
executable_path,
destination,
source,
experiment_name,
companion_file)
elif arguments.remove:
if arguments.cluster:
slurm_manager.remove("cluster", arguments.get("CLUSTER_NAME"))
if arguments.job:
slurm_manager.remove("job", arguments.get("JOB_NAME"))
elif arguments.list:
max_depth = 1 if arguments.get("DEPTH") is None else int(arguments.get("DEPTH"))
if arguments.get("clusters"):
slurm_manager.list("clusters", max_depth)
elif arguments.get("jobs"):
slurm_manager.list("jobs", max_depth)
elif arguments.set:
if arguments.get("cluster"):
cluster_name = arguments.get("CLUSTER_NAME")
parameter = arguments.get("PARAMETER")
value = arguments.get("VALUE")
slurm_manager.set_param("cluster", cluster_name, parameter, value)
if arguments.job:
config_name = arguments.get("JOB_NAME")
parameter = arguments.get("PARAMETER")
value = arguments.get("VALUE")
slurm_manager.set_param("job-metadata", config_name, parameter, value)
elif arguments.start and arguments.job:
job_name = arguments.get("JOB_NAME")
slurm_manager.run(job_name)
elif arguments.get("fetch"):
job_name = arguments.get("JOB_NAME")
slurm_manager.fetch(job_name)
elif arguments.test:
cluster_name = arguments.get("CLUSTER_NAME")
slurm_manager.connection_test(cluster_name)
elif arguments.clean:
job_name = arguments.get("JOB_NAME")
slurm_manager.clean_remote(job_name)
def do_image(self, args, arguments):
"""
::
Usage:
image list [NAMES] [--cloud=CLOUD] [--refresh] [--output=OUTPUT]
Options:
--output=OUTPUT the output format [default: table]
--cloud=CLOUD the cloud name
--refresh live data taken from the cloud
Description:
cm image list
cm image list --output=csv
cm image list 58c9552c-8d93-42c0-9dea-5f48d90a3188 --refresh
"""
map_parameters(arguments, "refresh", "cloud", "output")
VERBOSE.print(arguments, verbose=9)
variables = Variables()
if arguments.list and arguments.refresh:
names = []
clouds, names = Arguments.get_cloud_and_names(
"list", arguments, variables)
print("AAA", clouds, names)
for cloud in clouds:
print(f"cloud {cloud}")
provider = Provider(name=cloud)
images = provider.images()
order = provider.p.output['vm']['order'] # not pretty
header = provider.p.output['vm']['header'] # not pretty
print(
Printer.flatwrite(images,
sort_keys=["name"],
order=order,
header=header,
output=arguments.output))
return ""
elif arguments.list:
names = []
clouds, names = Arguments.get_cloud_and_names(
"list", arguments, variables)
print(clouds, names)
try:
for cloud in clouds:
print(f"List {cloud}")
p = Provider(cloud)
kind = p.kind
collection = "{cloud}-image".format(cloud=cloud,
kind=p.kind)
db = CmDatabase()
vms = db.find(collection=collection)
order = p.p.output['vm']['order'] # not pretty
header = p.p.output['vm']['header'] # not pretty
print(
Printer.flatwrite(vms,
sort_keys=["name"],
order=order,
header=header,
output=arguments.output))
except Exception as e:
VERBOSE.print(e, verbose=9)
return ""
def do_vm(self, args, arguments):
"""
::
Usage:
vm ping [NAMES] [--cloud=CLOUDS] [--count=N] [--processors=PROCESSORS]
vm check [NAMES] [--cloud=CLOUDS] [--processors=PROCESSORS]
vm status [NAMES] [--cloud=CLOUDS]
vm console [NAME] [--force]
vm start [NAMES] [--cloud=CLOUD] [--dryrun]
vm stop [NAMES] [--cloud=CLOUD] [--dryrun]
vm terminate [NAMES] [--cloud=CLOUD] [--dryrun]
vm delete [NAMES] [--cloud=CLOUD] [--dryrun]
vm refresh [--cloud=CLOUDS]
vm list [NAMES]
[--cloud=CLOUDS]
[--output=OUTPUT]
[--refresh]
vm boot [--name=NAME]
[--cloud=CLOUD]
[--username=USERNAME]
[--image=IMAGE]
[--flavor=FLAVOR]
[--public]
[--secgroup=SECGROUPs]
[--key=KEY]
[--dryrun]
vm boot [--n=COUNT]
[--cloud=CLOUD]
[--username=USERNAME]
[--image=IMAGE]
[--flavor=FLAVOR]
[--public]
[--secgroup=SECGROUPS]
[--key=KEY]
[--dryrun]
vm run [--name=NAMES] [--username=USERNAME] [--dryrun] COMMAND
vm script [--name=NAMES] [--username=USERNAME] [--dryrun] SCRIPT
vm ip assign [NAMES]
[--cloud=CLOUD]
vm ip show [NAMES]
[--group=GROUP]
[--cloud=CLOUD]
[--output=OUTPUT]
[--refresh]
vm ip inventory [NAMES]
vm ssh [NAMES] [--username=USER]
[--quiet]
[--ip=IP]
[--key=KEY]
[--command=COMMAND]
[--modify-knownhosts]
vm rename [OLDNAMES] [NEWNAMES] [--force] [--dryrun]
vm wait [--cloud=CLOUD] [--interval=SECONDS]
vm info [--cloud=CLOUD]
[--output=OUTPUT]
vm username USERNAME [NAMES] [--cloud=CLOUD]
vm resize [NAMES] [--size=SIZE]
Arguments:
OUTPUT the output format
COMMAND positional arguments, the commands you want to
execute on the server(e.g. ls -a) separated by ';',
you will get a return of executing result instead of login to
the server, note that type in -- is suggested before
you input the commands
NAME server name. By default it is set to the name of last vm from database.
NAMES server name. By default it is set to the name of last vm from database.
KEYPAIR_NAME Name of the vm keypair to be used to create VM. Note this is
not a path to key.
NEWNAMES New names of the VM while renaming.
OLDNAMES Old names of the VM while renaming.
Options:
--output=OUTPUT the output format [default: table]
-H --modify-knownhosts Do not modify ~/.ssh/known_hosts file
when ssh'ing into a machine
--username=USERNAME the username to login into the vm. If not
specified it will be guessed
from the image name and the cloud
--ip=IP give the public ip of the server
--cloud=CLOUD give a cloud to work on, if not given, selected
or default cloud will be used
--count=COUNT give the number of servers to start
--detail for table, a brief version
is used as default, use this flag to print
detailed table
--flavor=FLAVOR give the name or id of the flavor
--group=GROUP give the group name of server
--secgroup=SECGROUP security group name for the server
--image=IMAGE give the name or id of the image
--key=KEY specify a key to use, input a string which
is the full path to the private key file
--keypair_name=KEYPAIR_NAME Name of the vm keypair to
be used to create VM.
Note this is not a path to key.
--user=USER give the user name of the server that you want
to use to login
--name=NAME give the name of the virtual machine
--force rename/ delete vms without user's confirmation
--command=COMMAND
specify the commands to be executed
Description:
commands used to boot, start or delete servers of a cloud
vm default [options...]
Displays default parameters that are set for vm boot either
on the default cloud or the specified cloud.
vm boot [options...]
Boots servers on a cloud, user may specify flavor, image
.etc, otherwise default values will be used, see how to set
default values of a cloud: cloud help
vm start [options...]
Starts a suspended or stopped vm instance.
vm stop [options...]
Stops a vm instance .
vm delete [options...]
Delete servers of a cloud, user may delete a server by its
name or id, delete servers of a group or servers of a cloud,
give prefix and/or range to find servers by their names.
Or user may specify more options to narrow the search
vm floating_ip_assign [options...]
assign a public ip to a VM of a cloud
vm ip show [options...]
show the ips of VMs
vm ssh [options...]
login to a server or execute commands on it
vm list [options...]
same as command "list vm", please refer to it
vm status [options...]
Retrieves status of last VM booted on cloud and displays it.
vm refresh [--cloud=CLOUDS]
this command refreshes the data for virtual machines,
images and flavors for the specified clouds.
vm ping [NAMES] [--cloud=CLOUDS] [--count=N] [--processors=PROCESSORS]
pings the specified virtual machines, while using at most N pings.
The ping is executed in parallel.
If names are specifies the ping is restricted to the given names in
parameter format. If clouds are specified, names that are not in
these clouds are ignored. If the name is set in the variables
this name is used.
Tip:
give the VM name, but in a hostlist style, which is very
convenient when you need a range of VMs e.g. sample[1-3]
=> ['sample1', 'sample2', 'sample3']
sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18']
Quoting commands:
cm vm login gvonlasz-004 --command=\"uname -a\"
Limitations:
Azure: rename is not supported
"""
map_parameters(arguments, 'active', 'cloud', 'command', 'dryrun',
'flavor', 'force', 'output', 'group', 'image',
'interval', 'ip', 'key', 'modify-knownhosts', 'n',
'name', 'public', 'quiet', 'secgroup', 'size',
'username')
VERBOSE.print(arguments, verbose=9)
variables = Variables()
if arguments.refresh:
names = []
clouds, names = Arguments.get_cloud_and_names(
"refresh", arguments, variables)
return ""
elif arguments.ping:
# TODO: IMPLEMENT
names = []
pings = int(arguments.N or 3)
names = []
clouds, names = Arguments.get_cloud_and_names(
"ping", arguments, variables)
for name in names:
ping = Shell.live("ping -c {N} {name}".format(name=name,
N=arguments.N))
print(ping)
else:
return True
return ""
elif arguments.check:
names = []
clouds, names = Arguments.get_cloud_and_names(
"check", arguments, variables)
return ""
elif arguments.status:
names = []
clouds, names = Arguments.get_cloud_and_names(
"status", arguments, variables)
return ""
elif arguments.start:
names = []
clouds, names = Arguments.get_cloud_and_names(
"start", arguments, variables)
return ""
elif arguments.stop:
names = []
clouds, names = Arguments.get_cloud_and_names(
"stop", arguments, variables)
return ""
elif arguments.terminate:
names = []
clouds, names = Arguments.get_cloud_and_names(
"terminate", arguments, variables)
return ""
elif arguments.delete:
clouds, names = Arguments.get_cloud_and_names(
"delete", arguments, variables)
return ""
elif arguments.boot:
print("boot the vm")
elif arguments.list:
# vm list [NAMES]
# [--cloud=CLOUDS]
# [--output=OUPTUT]
# [--refresh]
# if no clouds find the clouds of all specified vms by name
# find all vms of the clouds,
# print only those vms specified by name, if no name is given print all for the cloud
# print("list the vms")
clouds, names = Arguments.get_cloud_and_names(
"list", arguments, variables)
# print("Clouds:", clouds)
if arguments.NAMES is not None:
names = Parameter.expand(arguments.NAMES)
Console.error("NAMES, not yet implemented" + str(names))
try:
if arguments["--refresh"]:
pass
# find all clouds in db
# iterate over the clouds
# for each name in name queue, find it and add it to the cloud vm list
# for each cloud print the vms
else:
pass
# find all clouds in db
# iterate over all clouds
# find the vm with the name
# add it to the cloud list
# for each cloud print the vms
except Exception as e:
VERBOSE.print(e, verbose=9)
return ""
else:
try:
if arguments["--refresh"]:
for cloud in clouds:
Console.ok("refresh " + cloud)
p = Provider(cloud)
vms = p.list()
order = p.p.output['vm']['order'] # not pretty
header = p.p.output['vm']['header'] # not pretty
print(
Printer.flatwrite(vms,
sort_keys=["cm.name"],
order=order,
header=header,
output=arguments.output))
else:
for cloud in clouds:
p = Provider(cloud)
kind = p.kind
# pprint(p.__dict__)
# pprint(p.p.__dict__) # not pretty
collection = "{cloud}-node".format(cloud=cloud,
kind=p.kind)
db = CmDatabase()
vms = db.find(collection=collection)
# pprint(vms)
# print(arguments.output)
# print(p.p.output['vm'])
order = p.p.output['vm']['order'] # not pretty
header = p.p.output['vm']['header'] # not pretty
print(
Printer.flatwrite(vms,
sort_keys=["cm.name"],
order=order,
header=header,
output=arguments.output))
except Exception as e:
VERBOSE.print(e, verbose=9)
return ""
elif arguments.info:
"""
vm info [--cloud=CLOUD] [--output=OUTPUT]
"""
print("info for the vm")
cloud, names = Arguments.get_cloud_and_names(
"info", arguments, variables)
elif arguments.rename:
print("rename the vm")
v = Variables()
cloud = v["cloud"]
p = Provider(cloud)
try:
oldnames = Parameter.expand(arguments["OLDNAMES"])
newnames = Parameter.expand(arguments["NEWNAMES"])
force = arguments["--force"]
if oldnames is None or newnames is None:
Console.error("Wrong VMs specified for rename",
traceflag=False)
elif len(oldnames) != len(newnames):
Console.error("The number of VMs to be renamed is wrong",
traceflag=False)
else:
print(oldnames)
print(newnames)
for i in range(0, len(oldnames)):
oldname = oldnames[i]
newname = newnames[i]
if arguments["--dryrun"]:
Console.ok("Rename {} to {}".format(
oldname, newname))
else:
print(f"rename {oldname} -> {newname}")
p.rename(source=oldname, destination=newname)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error("Problem renameing instances", traceflag=True)
elif arguments["ip"] and arguments["show"]:
print("show the ips")
"""
vm ip show [NAMES]
[--group=GROUP]
[--cloud=CLOUD]
[--output=OUTPUT]
[--refresh]
"""
elif arguments["ip"] and arguments["assign"]:
"""
vm ip assign [NAMES] [--cloud=CLOUD]
"""
print("assign the public ip")
elif arguments["ip"] and arguments["inventory"]:
"""
vm ip inventory [NAMES]
"""
print("list ips that could be assigned")
elif arguments.username:
"""
vm username USERNAME [NAMES] [--cloud=CLOUD]
"""
print("sets the username for the vm")
elif arguments.default:
print("sets defaults for the vm")
elif arguments.run:
"""
vm run [--name=NAMES] [--username=USERNAME] [--dryrun] COMMAND
"""
pass
elif arguments.script:
"""
vm script [--name=NAMES] [--username=USERNAME] [--dryrun] SCRIPT
"""
pass
elif arguments.resize:
"""
vm resize [NAMES] [--size=SIZE]
"""
pass
elif arguments.ssh:
"""
vm ssh [NAMES] [--username=USER]
[--quiet]
[--ip=IP]
[--key=KEY]
[--command=COMMAND]
[--modify-knownhosts]
"""
print("ssh the vm")
elif arguments.console:
# vm console [NAME] [--force]
names = Arguments.get_names(arguments, variables)
for name in names:
# r = vm.console(name,force=argument.force)
Console.msg("{label} {name}".format(label="console",
name=name))
return
elif arguments.wait:
"""
vm wait [--cloud=CLOUD] [--interval=SECONDS]
"""
print("waits for the vm till its ready and one can login")
def _execute(self, command):
VERBOSE.print(command, verbose=9)
os.system(command)
def do_queue(self, args, arguments):
"""
::
Usage:
queue create --name=NAME --policy=POLICY --cluster=CLUSTER
[--charge=CHARGE]
[--unit=UNIT]
queue activate [--name=NAME]
queue deactivate [--name=NAME]
queue set unit
queue connection_test --job=JOB
queue cluster list [--cluster=CLUSTERS] [--depth=DEPTH]
queue cluster remove [--cluster=CLUSTERS]
queue cluster set [--cluster=CLUSTERS] PARAMETER=VALUE
Arguments:
FILE a file name
INPUT_TYPE tbd
Options:
-f specify the file
--depth=DEPTH [default: 1]
--format=FORMAT [default: table]
Description:
This command creates a queue that is associated with a cloud.
We assume that a number of experiments are conducted with possibly
running the script multiple times. Each experiment will save the
batch script in its own folder.
The output of the script can be saved in a destination folder. A virtual
directory is used to coordinate all saved files.
The files can be located due to the use of the virtual directory on
multiple different data or file services
Authentication to the Batch systems is done viw the underlaying HPC
center authentication. We assume that the user has an account to
submit on these systems.
(SSH, 2 factor, XSEDE-account) TBD.
Examples:
LOTS OF DOCUMENTATION MISSING HERE
[--companion-file=COMPANION_FILE]
[--outfile-name=OUTPUT_FILE_NAME]
[--suffix=SUFFIX] [--overwrite]
"""
#
# create slurm manager so it can be used in all commands
#
queue = Queue() # debug=arguments["--debug"])
# arguments["--cloud"] = "test"
# arguments["NAME"] = "fix"
# map_parameters(arguments,
# "cloud",
# "name",
# "cluster",
# "script",
# "type",
# "destination",
# "source",
# "format")
# if not arguments.create
# find cluster name from Variables()
# if no cluster is defined look it up in yaml in batch default:
# if not defined there fail
# clusters = Parameter.expand(arguments.cluster)
# name = Parameters.expand[argumnets.name)
# this will return an array of clusters and names of jobs and all cluster
# job or clusterc commands will be executed on them
# see the vm
#
# if active: False in the yaml file for the cluster this cluster is not used and scipped.
VERBOSE.print(arguments, verbose=9)
implemented_policies = ['FIFO', 'FILO']
variables = Variables()
# docopt for some reason does not show all of the arguments in dot
# format that's the reason I used -- format.
if arguments.create and \
arguments['--name'] and \
arguments['--cluster'] and \
arguments['--policy']:
queue_name = arguments['--name']
cluster_name = arguments['--cluster']
policy = arguments['--policy']
if policy.upper() not in ['FIFO', 'FILO']:
Console.error("Policy {policy} not defined, currently "
"implemented policies are {policies} ".format(
policy=policy.upper(),
policies=implemented_policies))
return
charge = arguments['--charge']
unit = arguments['--unit']
queue.create(queue_name, cluster_name, policy, charge, unit)