def test_variables_assign(self):
HEADING("assign key=value")
v = Variables()
n = len(v)
v["gregor"] = "gregor"
assert (len(v) == n + 1)
assert "gregor" in v
v.close()
def test_test_variable_remove(self):
HEADING("directory and key subtract ")
d = {"a": "1", "b": "2"}
v = Variables()
v + d
print(v)
assert "a" in v and "b" in v
v - d.keys()
assert "a" not in v and "b" not in v
print(v)
v.close()
def postcmd(self, stop, line):
StopWatch.stop("command")
try:
variable = Variables()
if "timer" not in variable:
variable["timer"] = "False"
if str(variable["timer"].lower()) in ['on', 'true']:
print("Timer: {:.4f}s ({})".format(StopWatch.get("command"),
line.strip()))
variable.close()
except Exception as e:
Error.traceback(error=e)
return stop
def setup(self):
variables = Variables()
print(variables['storage'])
self.service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=self.service)
self.sourcedir = path_expand("~/.cloudmesh/storage/test")
print()
def VERBOSE(msg, label=None, color="BLUE", verbose=9, location=True):
"""
Prints a data structure in verbose mode
:param msg: the msg to be printed, can be a datastructure suc as a dict
:param label: the label to be used, defaults to the name of the msg variable
:param color: the color
:param verbose: indicates when to print it. If verbose in cloudmesh is higher than the speified value it is printed
:return:
"""
_verbose = int(Variables()["verbose"] or 0)
if _verbose >= verbose:
if label is None:
label = inspect.stack()[1][4][0].strip().replace("VERBOSE(", "")
label = label.split(",")[0][:-1]
if location:
cwd = os.getcwd()
frame = inspect.stack()[1]
module = inspect.getmodule(frame[0])
filename = module.__file__.replace(cwd, ".")
lineno = str(inspect.stack()[1][2]) + ":" + str(
inspect.stack()[1][3])
print("# FILENAME:", filename, sep="")
hline = "\n" + 70 * "-" + "\n"
banner(filename + hline + lineno + hline + pformat(msg),
label=label,
color=color)
def get(self, name=None):
connection = None
if name is None:
variables = Variables()
# noinspection PyUnusedLocal
cloudname = variables['cloud']
kind = self.config.get(
"cloudmesh.cloud.{name}.cm.kind".format(name=name))
credentials = self.config.get(
"cloudmesh.cloud.{name}.credentials".format(name=name))
# BUG FROM HERE ON WRONG
if kind == 'azure':
AZDriver = get_driver(Provider.AZURE)
connection = AZDriver(
subscription_id=credentials['AZURE_SUBSCRIPTION_ID'],
key_file=credentials['AZURE_MANAGEMENT_CERT_PATH'])
elif kind == 'aws':
EC2Driver = get_driver(Provider.EC2)
connection = EC2Driver(credentials['EC2_ACCESS_ID'],
credentials['EC2_SECRET_KEY'])
return connection
def __init__(self,
service=None,
json=None,
**kwargs):
super().__init__(service=service)
variables=Variables()
self.debug=variables['debug']
if json:
self.path = path_expand(json)
self.client = storage.Client.from_service_account_json(self.path)
else:
self.config = Config()
self.configuration = self.config[f"cloudmesh.storage.{service}"]
self.kind = self.config[f"cloudmesh.storage.{service}.cm.kind"]
self.credentials = dotdict(self.configuration["credentials"])
self.bucket_name = self.config[f"cloudmesh.storage.{service}.default.directory"]
# self.yaml_to_json(service)
self.path = path_expand("~/.cloudmesh/google.json")
#print("11111:",self.path)
#print("bucketName:", self.bucket_name)
self.client = storage.Client.from_service_account_json(self.path) #Important for goole login
self.storage_dict = {}
self.bucket = self.client.get_bucket(self.bucket_name)
def set_debug(self, on):
def _boolean(value):
value = str(value).lower()
if value in ["true", "on"]:
value = True
elif value in ["false", "off"]:
value = False
else:
value = False
Console.error("Value is not boolean")
return value
variables = Variables()
if _boolean(on):
variables["debug"] = True
variables["trace"] = True
variables["verbose"] = '10'
variables["timer"] = True
else:
variables["debug"] = False
variables["trace"] = False
variables["verbose"] = '0'
variables["timer"] = False
for key in ["debug", "trace", "verbose", "timer"]:
print(f"{key}={variables[key]}")
def boot(self, order='price', refresh=False, cloud=None):
clouds = ['aws', 'azure', 'gcp']
if cloud in clouds:
clouds = [cloud]
Console.msg(f"Checking to see which providers are bootable ...")
reachdict = {}
for cloud in clouds:
try:
tempProv = Provider(
name=cloud, configuration="~/.cloudmesh/cloudmesh.yaml")
Console.msg(cloud + " reachable ...")
reachdict[cloud] = tempProv
except:
Console.msg(cloud + " not available ...")
flavorframe = self.list(order, 10000000, refresh, printit=False)
keysya = list(reachdict.keys())
flavorframe = flavorframe[flavorframe['provider'].isin(keysya)]
Console.msg(f"Showing top 5 options, booting first option now...")
converted = flavorframe.head(5).to_dict('records')
print(Printer.write(converted))
cheapest = converted[0]
var_list = Variables(filename="~/.cloudmesh/var-data")
var_list['cloud'] = cheapest['provider']
Console.msg(f'new cloud is ' + var_list['cloud'] +
', booting up the vm with flavor ' +
cheapest['machine-name'])
vmcom = VmCommand()
vmcom.do_vm('boot --flavor=' + cheapest['machine-name'])
return ""
def test_cli_list(self):
HEADING()
r = _run("cms var list")
v = Variables()
print(r)
assert v['deleteme'] == 'abc'
assert "deleteme='abc'" in r
def test_cli_delete(self):
r = run("cms var delete deleteme")
v = Variables()
print("Result:", r)
print("Variable:", v)
assert v['deleteme'] != 'abc'
def test_variables_add(self):
HEADING("directory add ")
d = {"a": "1", "b": "2"}
v = Variables()
v + d
print(v)
assert "a" in v and "b" in v
del v["a"]
del v["b"]
v + d
assert "a" in v and "b" in v
v - d
assert "a" not in v and "b" not in v
print(v)
v.close()
def create(self, **kwargs):
try:
data = self.provider.create(**kwargs)
variables = Variables()
variables["volume"] = data["cm"]["name"]
except:
raise ValueError("Volume could not be created")
return data
def create_vm(name):
variables = Variables()
variables['debug'] = True
variables['trace'] = True
variables['verbose'] = 10
banner(f"running vm: {name}")
def __init__(self, user=None):
self.hostname = "juliet.futuresystems.org"
self.host = "juliet"
self.user = user
variables = Variables()
self.debug = variables["debug"]
if user is not None:
self.login = f"{user}@{self.host}"
def test_setup(self):
self.variables = Variables()
self.storages = Parameter.expand(self.variables['storage'])
pytest.storage = self.storages[0]
command = ['python', 'server.py']
pytest.openapi = run(command)
pytest.openapi.execute()
print(pytest.openapi.pid)
time.sleep(5)
def VERBOSE(msg,
label=None,
color="BLUE",
verbose=9,
location=True,
secrets=[
"OS_PASSWORD", "OS_USERNAME", "client_secret", "client_id",
"project_id", "AZURE_TENANT_ID", "AZURE_SUBSCRIPTION_ID",
"AZURE_APPLICATION_ID", "AZURE_SECRET_KEY: TBD",
"EC2_ACCESS_ID: TBD", "EC2_SECRET_KEY", "MONGO_PASSWORD"
]):
"""
Prints a data structure in verbose mode
:param msg: the msg to be printed, can be a data structure such as a dict
:param label: the label to be used, defaults to the name of the msg variable
:param color: the color
:param verbose: indicates when to print it. If verbose in cloudmesh is
higher than the specified value it is printed
:return:
"""
_verbose = int(Variables()["verbose"] or 0)
if _verbose >= verbose:
verbose_lock.acquire()
if label is None:
label = inspect.stack()[1][4][0].strip().replace("VERBOSE(", "")
label = label.split(",")[0][:-1]
if location:
cwd = os.getcwd()
frame = inspect.stack()[1]
module = inspect.getmodule(frame[0])
filename = module.__file__.replace(cwd, ".")
lineno = str(inspect.stack()[1][2]) + ":" + str(
inspect.stack()[1][3])
# print("# FILENAME:", filename, sep="")
hline = "\n" + 70 * "-" + "\n"
if type(msg) == dict and secrets is not None:
tmp = dict(msg)
for key in secrets:
if key in tmp:
tmp[key] = "********"
banner(lineno + " " + filename + hline + pformat(tmp),
label=label,
color=color)
else:
banner(lineno + " " + filename + hline + pformat(msg),
label=label,
color=color)
verbose_lock.release()
def postcmd(self, stop, line):
StopWatch.stop("command")
try:
variable = Variables()
if "timer" not in variable:
variable["timer"] = "False"
if str(variable["timer"].lower()) in ['on', 'true']:
command_time = StopWatch.get("command")
load_time = StopWatch.get("load")
line_strip = line.strip()
print(f"# Timer: {command_time:.4f}s "
f"Load: {load_time:.4f}s "
f"{line_strip}")
variable.close()
except Exception as e:
Error.traceback(error=e)
return stop
def __getitem__(self, key):
try:
return self.data[key]
except:
name = f"cloudmesh.compute.{key}.Provider"
provider = __import__(name)
self.load()
variables = Variables()
if variables['debug'] is 'False':
Console.cprint("BLUE", "", f"Loading Provider: '{key}'")
return self.data[key]
def f(test):
msg = "This is a test {test}".format(**locals())
print(" jj ", locals())
from cloudmesh.common.debug import VERBOSE
d = {'test': 'Gergor'}
VERBOSE(d, "a", "RED", 100)
from cloudmesh.common.console import Console
msg = 'my message'
Console.ok(msg) # prins a green message
Console.error(msg) # prins a red message proceeded with ERROR
Console.msg(msg) # prins a regular black message
from cloudmesh.common.variables import Variables
variables = Variables()
variables['debug'] = True
variables['trace'] = True
variables['verbose'] = 10
m = {'key': 'value'}
VERBOSE(m)
a = {'p': "ac"}
print(a['p'])
from cloudmesh.common.Shell import Shell
result = Shell.execute('pwd')
print(result)
result = Shell.execute('ls', ['-l', '-a'])
print(result)
result = Shell.execute('ls', '-l -a')
print(result)
result = Shell.ls('-aux')
print(result)
result = Shell.ls('-a')
print(result)
result = Shell.pwd()
print(result)
from cloudmesh.common.StopWatch import StopWatch
from time import sleep
StopWatch.start('test')
sleep(1)
StopWatch.stop('test')
print(StopWatch.get('test'))
def replace_vars(self, line):
# self.update_time()
variable = Variables()
newline = line
if len(variable) is not None:
for name in variable.data:
value = str(variable[name])
newline = newline.replace("$" + name, value)
newline = newline.replace("var." + name, value)
for v in os.environ:
name = v.replace('os.', '')
if name in newline:
value = os.environ[name]
newline = newline.replace("os." + v, value)
default = Default()
if default is not None:
for v in default.data:
name = "default." + v.replace(",", ".")
value = default.data[v]
if name in newline:
newline = newline.replace(name, value)
# replace if global is missing
global_default = default["global"]
if global_default is not None:
for v in global_default:
name = "default." + v
value = global_default[v]
if name in newline:
newline = newline.replace(name, value)
default.close()
variable.close()
newline = path_expand(newline)
return line, newline
def debug():
"""
sets the cms shell variables
trace = True
debug = True
verbose = 10
"""
variables = Variables()
variables["trace"] = True
variables["debug"] = True
variables["verbose"] = 10
def load(self, config_path=None):
"""
loads a configuration file
:param config_path:
:type config_path:
:return:
:rtype:
"""
# VERBOSE("Load config")
self.config_path = Path(path_expand(config_path or self.location.config())).resolve()
self.config_folder = dirname(self.config_path)
self.create(config_path=config_path)
with open(self.config_path, "r") as stream:
content = stream.read()
# content = path_expand(content)
content = self.spec_replace(content)
self.data = yaml.load(content, Loader=yaml.SafeLoader)
# print (self.data["cloudmesh"].keys())
# self.data is loaded as nested OrderedDict, can not use set or get
# methods directly
if self.data is None:
raise EnvironmentError(
"Failed to load configuration file cloudmesh.yaml, "
"please check the path and file locally")
#
# populate default variables
#
self.variable_database = Variables(filename="~/.cloudmesh/variable.dat")
self.set_debug_defaults()
default = self.default()
for name in self.default():
if name not in self.variable_database:
self.variable_database[name] = default[name]
if "cloud" in default:
self.cloud = default["cloud"]
else:
self.cloud = None
def detach(self, name=None):
"""
Detach volumes from vm.
If success, the last volume will be saved as the most recent volume.
:param names: names of volumes to detach
:return: dict
"""
try:
result = self.provider.detach(name)
variables = Variables()
variables["volume"] = result["cm"]["name"]
except:
raise ValueError("Volume could not be detached")
return result
def __init__(self, config=None):
self.hostname = "juliet.futuresystems.org"
self.tunnel_host = "juliet"
self.user = config["user"]
self.port = config["port"]
self.host = config["host"]
self.gpu = config["gpu"]
variables = Variables()
self.debug = variables["debug"]
self.logfile = "jupyterlab.log"
if self.user is not None:
self.login = f"{self.user}@{self.tunnel_host}"
def setup(self):
StopWatch.start("vdir setup")
self.vdir = Vdir()
self.endpoint = 'box:/test.txt'
self.directory_and_name = '/testdir/test'
self.directory = 'testdir'
self.file = 'test'
self.create_file('~/.cloudmesh/vdir/test/test.txt', 'test file')
self.destination = path_expand("~/.cloudmesh/vdir/test")
variables = Variables()
service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=service)
self.p.put(source='~/.cloudmesh/vdir/test/test.txt', destination='/',
recursive=False)
StopWatch.stop("vdir setup")
def test_cli_set(self):
HEADING()
r = _run("cms var deleteme=abc")
print(r)
data = path_expand("~/.cloudmesh/variables.dat")
cat = _run(f"cat {data}")
print(cat)
assert "deleteme: abc" in cat
v = Variables()
print("Data", v.__dict__["data"].__dict__)
value = v['deleteme']
print("Value:", value)
assert value == 'abc'
def do_debug(self, args, arguments):
"""
::
Usage:
debug on
debug off
Description:
debug on
sets the variables
debug=True
trace=True
verbose=10
timer=True
debug off
sets the variables
debug=False
trace=False
verbose=0
timer=False
"""
database = Variables(filename="~/.cloudmesh/var-data")
if arguments.on:
database["debug"] = True
database["trace"] = True
database["verbose"] = '10'
database["timer"] = True
elif arguments.off:
database["debug"] = False
database["trace"] = False
database["verbose"] = '0'
database["timer"] = False
def create(self, **kwargs):
"""
Create a volume.
:param name (string): name of volume
:param region (string): availability-zone
:param size (integer): size of volume
:param volume_type (string): type of volume.
:param description (string)
:return: dict
"""
try:
data = self.provider.create(**kwargs)
variables = Variables()
variables["volume"] = data[0]["cm"]["name"]
except:
raise ValueError("Volume could not be created")
return data
def add_tag(self, **kwargs):
"""
This function add tag to a volume.
If name is not specified, then tag will be added to the last volume.
If success, the volume will be saved as the most recent volume.
:param name: name of volume
:param kwargs:
key: name of tag
value: value of tag
:return: self.list()
"""
try:
result = self.provider.add_tag(**kwargs)
variables = Variables()
variables["volume"] = result["cm"]["name"]
except:
raise ValueError("Tag could not be added")
return result
