def defaults():
"""
default values
:return: a number of default values for memory, image, and script
:rtype: dotdict
"""
# TODO, read this from yaml file
d = dotdict()
d.memory = 1024
# d.image = "ubuntu/xenial64"
d.image = "ubuntu/trusty64"
d.port = 8080
d.script = None
return d
def create(self, **kwargs):
con = azure.connect(**self.config)
arguments = dotdict(kwargs)
if arguments.volume_type == None:
arguments.volume_type = self.defaults["volume_type"]
if arguments.size == None:
arguments.size = self.defaults["size"]
print(arguments.NAME)
con.create_volume(name=arguments.NAME, size=arguments.size,
volume_type=arguments.volume_type)
# print list after create
results = con.list_volumes()
result = self.update_dict(results)
print(self.Print(result, kind='volume', output=kwargs['output']))
def delete(self, name=None):
# TODO: check
arg = dotdict()
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"]["vagrant"]["default"][
"path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
result = Shell.execute("vagrant", ["destroy", "-f", name],
cwd=arg.directory)
return result
def dotdict_demo():
"""
dotdict enables access python dictionary by dot
"""
dict_demo = ({"key": "value"})
# In python the value of a dictionary is retrieved by
# using square brackets with subscriptions
print(dict_demo['key'])
# Using dotdict will enable the access by dots
ddict = dotdict(dict_demo)
print(ddict.key)
return
def server_dotdict(self, serverCommand) -> dotdict:
if serverCommand.__contains__("generate"):
words = serverCommand.split()
dotdictd = dotdict()
for word in words:
if "--import_class" in word:
dotdictd["import_class"] = True
dotdictd["all_functions"] = False
elif "--all_functions" in word:
dotdictd["all_functions"] = True
dotdictd["import_class"] = False
elif "--filename" in word:
dotdictd["--filename"] = word.split("=")[1]
return dotdictd
if serverCommand.__contains__("server"):
pass
def __init__(self):
self.__dict__ = self.__shared_state
if "data" not in self.__dict__:
self.config = Config()
self.data = dotdict(self.config.data["cloudmesh"]["data"]["mongo"])
self.expanduser()
if self.data.MONGO_PASSWORD in ["TBD", "admin"]:
Console.error("MongoDB password must not be the default")
raise Exception("password error")
mongo_path = self.data["MONGO_PATH"]
mongo_log = self.data["MONGO_LOG"]
paths = [mongo_path, mongo_log]
for path in paths:
if not os.path.exists(path):
os.makedirs(path)
def execute(self, name, command, cwd=None):
arg = dotdict()
arg.cwd = cwd
arg.command = command
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"][cloud]["default"]["path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
vms = self.to_dict(self.nodes())
arg = "ssh {} -c {}".format(name, command)
result = Shell.execute("vagrant", ["ssh", name, "-c", command],
cwd=arg.directory)
return result
def class_artifact(repos, kind):
global community
headers = [
"hid", "firstname", "lastname", "community", "semester", "url", "kind",
"title", "Artifact"
]
t = []
for repo in repos:
artifact = ""
title = ""
try:
content = readfile(f"./../{repo}/README.yml")
data = yaml.load(content, Loader=yaml.SafeLoader)
owner = dotdict(data["owner"])
url = f"{community}/{repo}"
t.append([
owner.hid, owner.firstname, owner.lastname, owner.community,
owner.semester, url, kind, title, artifact
])
# print (data[kind])
if kind in data:
artifacts = data[kind]
for artifact in artifacts:
title = "TBD"
artifact_url = "TBD"
if "title" in artifact:
title = artifact["title"]
if "url" in artifact:
artifact_url = artifact["url"]
t.append([
owner.hid, owner.firstname, owner.lastname,
owner.community, owner.semester, url, kind, title,
artifact_url
])
t.append(["", "", "", "", "", "", "", "", ""])
except Exception as e:
# print (e)
# VERBOSE(repo)
pass
print()
print(tabulate(t, headers=headers))
print()
def live(cls, command, cwd=None):
if cwd is None:
cwd = os.getcwd()
process = subprocess.Popen(shlex.split(command),
cwd=cwd,
stdout=subprocess.PIPE)
result = b''
while True:
output = process.stdout.read(1)
if output == b'' and process.poll() is not None:
break
if output:
result = result + output
sys.stdout.write(output.decode("utf-8"))
sys.stdout.flush()
rc = process.poll()
data = dotdict({"status": rc, "text": output.decode("utf-8")})
return data
def do_info(self, args, arguments):
"""
::
Usage:
info [path|commands|files|cloudmesh]
Description:
info
provides internal info about the shell and its packages
"""
arguments = dotdict(arguments)
module_list = Plugin.modules()
if arguments.commands:
commands = Plugin.classes()
print_list(commands)
elif arguments.path:
path_list = cloudmesh.__path__
print_list(path_list)
elif arguments.files:
commands = Plugin.modules()
for command in commands:
try:
r = inspect.getfile(command)
print("*", type(command))
except Exception as e:
print(e)
elif arguments.help:
for name in module_list:
p = "cloudmesh." + name
help_string = p + " not found."
# noinspection PyBroadException
try:
help_string = pydoc.render_doc(
p, "Help on %s" + "\n" + 79 * "=")
except Exception as e:
pass
print(help_string)
else:
print_list(module_list)
def class_list(repos, location):
global community
owners = []
for repo in repos:
try:
path = f"{location}/{repo}/README.yml"
path = Path(path).resolve()
content = readfile(path)
data = yaml.load(content, Loader=yaml.SafeLoader)
owner = dotdict(data["owner"])
owner["url"] = f"{community}/{repo}"
owners.append(owner)
except Exception as e:
print(e)
VERBOSE(repo)
return owners
def build_dotdict(self, function_name=None) -> dotdict:
dotdictd = dotdict()
if globalcommandparameter.import_class:
dotdictd["import_class"] = True
dotdictd["all_functions"] = False
dotdictd[
"FUNCTION"] = globalcommandparameter.module_name.capitalize()
elif globalcommandparameter.all_functions:
dotdictd["all_functions"] = True
dotdictd["import_class"] = False
else:
dotdictd["all_functions"] = False
dotdictd["import_class"] = False
dotdictd["FUNCTION"] = function_name
dotdictd[
"--filename"] = globalcommandparameter.module_directory + "/build/" + globalcommandparameter.module_name + ".py"
return dotdictd
def class_table(repos):
global community
headers = ["hid", "firstname", "lastname", "community", "semester", "url"]
t = []
for repo in repos:
try:
content = readfile(f"./../{repo}/README.yml")
data = yaml.load(content, Loader=yaml.SafeLoader)
owner = dotdict(data["owner"])
url = f"{community}/{repo}"
t.append(
[owner.hid, owner.firstname, owner.lastname, owner.community,
owner.semester, url])
except Exception as e:
print(e)
VERBOSE(repo)
print()
print(tabulate(t, headers=headers))
print()
def create_contributors(owners, location):
#
# BUG repo not defined
#
global community
print("# Contributors")
print()
t = []
headers = ["Hid", "Firstname", "Lastname", "Community", "Semester"]
for owner in owners:
owner = dotdict(owner)
hid = owner.hid
entry = [
f"[{hid}]({community}/{hid})", owner.lastname, owner.firstname,
owner.community, owner.semester
]
t.append(entry)
print(tabulate(t, headers=headers, tablefmt="github"))
def suspend(self, name=None):
"""
NOT IMPLEMENTED
suspends the node with the given name
:param name: the name of the node
:return: The dict representing the node
"""
# TODO: find last name if name is None
arg = dotdict()
arg.name = name
# TODO find the vbx name from vagrantname
arg.path = self.default["path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
result = Shell.execute("vbox", ["suspend", name], cwd=arg.directory, shell=True)
return result
def get_credentials():
credential_file = os.environ['CLOUDMESH_CREDENTIALS_YAML']
credentials_list = []
with open(credential_file) as file:
documents = yaml.full_load(file)
credentials = flatten(documents)
credentials = dotdict(credentials)
# add needed credentials to the credentials array
credentials_list.append(
credentials.cloudmesh__cloud__azure__credentials__AZURE_TRANSLATOR_KEY)
credentials_list.append(
credentials.
cloudmesh__cloud__azure__credentials__AZURE_TEXT_ANALYTICS_KEY)
credentials_list.append(
credentials.
cloudmesh__cloud__azure__credentials__AZURE_TEXT_ANALYTICS_ENDPOINT)
credentials_list.append(
credentials.cloudmesh__cloud__google__credentials__auth__json_file)
return credentials_list
def destroy(self, groupName=None, vmName=None):
"""
Destroys the node
:param name: the name of the node
:return: the dict of the node
"""
if groupName is None:
groupName = self.GROUP_NAME
if vmName is None:
vmName = self.VM_NAME
node = dotdict()
node.name = groupName
node.vmName = vmName
# Delete VM
print('\nDeleteing Azure Virtual Machine')
async_vm_delete = self.compute_client.virtual_machines.delete(
groupName, vmName)
async_vm_delete.wait()
return node
def do_commands(self, args, arguments):
"""
::
Usage:
commands
"""
arguments = dotdict(arguments)
print("Plugin.list()")
print(Plugin.list())
names = self.get_names()
print()
print("get_names()")
print(names)
print()
print("dir")
print(dir(cloudmesh))
def info(self):
data = dotdict({
"hostname": Shell.run("hostname"),
"repo": Shell.run("git config --get remote.origin.url"),
"python": Shell.run("which python"),
"user": Shell.run("whoami"),
"cwd": path_expand(os.curdir),
"home": str(Path.home()),
"port": self.port,
"tunnel": None
})
data.workdir = data.cwd.replace(f"{data.home}/", "")
if self.port is not None:
netstat = Shell.run(f"netstat -tulpen")
for line in netstat.splitlines():
if self.port in line and "127.0.0.1" in line:
data.tunnel = line.strip().split(" ")[-1].split("/")[0]
return data
def _get_specification(self,
cloud=None,
name=None,
port=None,
image=None,
**kwargs):
arg = dotdict(kwargs)
arg.port = port
config = Config()
pprint(self.config)
if cloud is None:
#
# TOD read default cloud
#
cloud = "vagrant" # TODO must come through parameter or set cloud
print("CCC", cloud)
spec = config.data["cloudmesh"]["cloud"][cloud]
pprint(spec)
default = spec["default"]
pprint(self.default)
if name is not None:
arg.name = name
else:
# TODO get new name
pass
if image is not None:
arg.image = image
else:
arg.image = default["image"]
pass
arg.path = default["path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
arg.vagrantfile = "{directory}/Vagrantfile".format(**arg)
return arg
def boot(cls, **kwargs):
arg = dotdict(kwargs)
arg.cwd = kwargs.get("cwd", None)
vms = cls.to_dict(cls.list())
if arg.name in vms:
Console.error("vm {name} already booted".format(**arg),
traceflag=False)
return None
# print result
else:
cls.create(**kwargs)
Console.ok("{name} created".format(**arg))
Console.ok("{name} booting ...".format(**arg))
result = Shell.execute("vagrant", ["up", arg.name], cwd=arg.name)
Console.ok("{name} ok.".format(**arg))
return result
def stop(self, groupName=None, vmName=None):
"""
stops the node with the given name
:param name:
:return: The dict representing the node including updated status
"""
if groupName is None:
groupName = self.GROUP_NAME
if vmName is None:
vmName = self.VM_NAME
node = dotdict()
node.name = groupName
node.vmName = vmName
# Stop the VM
print('\nStopping Azure VM')
async_vm_stop = self.compute_client.virtual_machines.power_off(
groupName, vmName)
async_vm_stop.wait()
return node
def vagrantfile(self, **kwargs):
arg = dotdict(kwargs)
provision = kwargs.get("script", None)
if provision is not None:
arg.provision = 'config.vm.provision "shell", inline: <<-SHELL\n'
for line in textwrap.dedent(provision).splitlines():
if line.strip() != "":
arg.provision += 12 * " " + " " + line + "\n"
arg.provision += 12 * " " + " " + "SHELL\n"
else:
arg.provision = ""
# not sure how I2 gets found TODO verify, comment bellow is not enough
# the 12 is derived from the indentation of Vagrant in the script
# TODO we may need not just port 80 to forward
script = textwrap.dedent("""
Vagrant.configure(2) do |config|
config.vm.define "{name}"
config.vm.hostname = "{name}"
config.vm.box = "{image}"
config.vm.box_check_update = true
config.vm.network "forwarded_port", guest: 80, host: {port}
config.vm.network "private_network", type: "dhcp"
# config.vm.network "public_network"
# config.vm.synced_folder "../data", "/vagrant_data"
config.vm.provider "virtualbox" do |vb|
# vb.gui = true
vb.memory = "{memory}"
end
{provision}
end
""".format(**arg))
return script
def start(self, groupName=None, vmName=None):
"""
start a node
:param name: the unique node name
:return: The dict representing the node
"""
if groupName is None:
groupName = self.GROUP_NAME
if vmName is None:
vmName = self.VM_NAME
node = dotdict()
node.name = groupName
node.vmName = vmName
# Start the VM
print('\nStarting Azure VM')
async_vm_start = self.compute_client.virtual_machines.start(
groupName, vmName)
async_vm_start.wait()
return node
def new(instance, args):
# instance.new.__doc__ = doc
# noinspection PyUnusedLocal
try:
argv = shlex.split(args)
# pprint(argv)
arguments = dotdict(docopt(doc, help=True, argv=argv))
# pprint(arguments)
verbose = int(Variables()["verbose"] or 0)
if verbose > 9:
s = pformat(arguments)
banner(s, label="Arguments", color="BLUE")
func(instance, args, arguments)
except SystemExit as e:
if args not in ('-h', '--help'):
if "::" in doc:
usage = textwrap.dedent(doc.split("::")[1])
else:
usage = doc
usage = textwrap.dedent(doc.split("Usage:")[1])
print()
print("Usage:")
for line in usage.split("\n"):
if ":" in line:
kind = line.split(":")[0]
if kind in [
"Arguments", "Options", "Example",
"Descriptiom"
]:
break
print(line)
Console.error(
"Could not execute the command. Please check usage with")
print()
Console.msg(" cms help", name.replace("do_", ""))
print()
def run(cmd):
out = dotdict({
"stdout": None,
"stderr": None,
"code": None,
})
os.environ['PYTHONUNBUFFERED'] = "1"
proc = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
proc.wait()
out.stdout, out.stderr = proc.communicate()
out.code = proc.returncode
if out.stderr is not None:
out.stderr = out.stderr.decode("utf-8")
if out.stdout is not None:
out.stdout = out.stdout.decode("utf-8")
return out
def resume(self, name=None):
"""
NOT IMPLEMENTED
Default: resume(start) all the VMs specified.
If @name is provided, only the named VM is started.
:param name: [optional], name of the Vagrant VM.
:return:
"""
# TODO: find last name if name is None
arg = dotdict()
arg.name = name
# TODO find the vbx name from vagrantname
arg.path = self.default["path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
result = Shell.execute("vbox", ["up", name], cwd=arg.directory,
shell=True)
return result
def __init__(self, service=None, json=None, **kwargs):
super().__init__(service=service)
variables = Variables()
self.debug = variables['debug']
if json:
self.path = path_expand(json)
self.client = storage.Client.from_service_account_json(self.path)
else:
self.config = Config()
self.configuration = self.config[f"cloudmesh.storage.{service}"]
self.kind = self.config[f"cloudmesh.storage.{service}.cm.kind"]
self.credentials = dotdict(self.configuration["credentials"])
self.bucket_name = self.config[
f"cloudmesh.storage.{service}.default.directory"]
# self.yaml_to_json(service)
self.path = path_expand("~/.cloudmesh/google.json")
#print("11111:",self.path)
#print("bucketName:", self.bucket_name)
self.client = storage.Client.from_service_account_json(
self.path) #Important for goole login
self.storage_dict = {}
self.bucket = self.client.get_bucket(self.bucket_name)
def default(self):
return dotdict(self.data["cloudmesh"]["default"])
# E.Cloudmesh.Common.2 - week 3
# Develop a program that demonstartes the use of dotdict
# from CM import dotdict fuction
from cloudmesh.common.dotdict import dotdict
info = {
"name": "Zhi",
"title": "Ph.D. candidate in public health",
"depart": "dept. of applied health science"
}
data = dotdict(info)
if data.name is "Zhi":
print("this is quite readable")
else:
print("this is non-readable")
