def generate_ssh_key(self, hostname):
Shell.execute('mkdir', f'-p {Configure.KEY_DIR}')
Shell.run(f'ssh-keygen -q -N "" -C "ubuntu@{hostname}" -f '
f'{Configure.KEY_DIR}/id_rsa')
priv_key = readfile(f'{Configure.KEY_DIR}/id_rsa').strip()
pub_key = readfile(f'{Configure.KEY_DIR}/id_rsa.pub').strip()
self.manager_public_key = pub_key
Shell.execute('rm', path_expand(f'{Configure.KEY_DIR}/id_rsa'))
Shell.execute('rm', path_expand(f'{Configure.KEY_DIR}/id_rsa.pub'))
return priv_key, pub_key
def _convert_file(self,
file=None,
command=None,
include=None,
tag="MANPAGE"):
file = file or "README.md"
tag_string = f"<!--{tag}-->"
try:
if command is not None:
man = Shell.run(f"cms help {command}")
elif include is not None:
man = readfile(include)
else:
man = "No command or file specified"
# remove timer
if "\n# Timer: " in man:
man = man.split("\n# Timer: ")[0]
readme = readfile(file)
parts = readme.split(tag_string)
content = []
content.append(parts[0].strip())
content.append("")
content.append(tag_string)
if command is not None:
content.append("```")
content.append(textwrap.dedent("\n".join(
man.splitlines()[7:])))
content.append("```")
else:
content.append(textwrap.dedent("\n".join(man.splitlines())))
content.append(tag_string)
try:
content.append(parts[2])
except:
pass
except Exception as e:
print(e)
content.append("")
content.append(tag_string)
content.append("```")
content.append(e)
content.append("```")
content.append(tag_string)
manpage = "\n".join(content).strip()
return manpage, man
def fix_keys_file(filename):
# concatenate ~/.ssh/id_rsa.pub
lines = readfile(filename)
key = readfile(path_expand("~/.ssh/id_rsa.pub"))
authorized_keys = lines + key
keys = ''.join(authorized_keys)
# remove duplicates and empty lines
keys_list = [x for x in list(set(keys.splitlines())) if x != '\n']
keys = ('\n'.join(keys_list) + '\n')
writefile(filename, str(keys))
def create_notebook():
kubeman.banner("create_notebook")
# port = notebook_port()
# ip = kubeman.Shell_run("minikube ip").strip()
token = kubeman.get_token()
print(token)
for file in [
# f"{CONTAINERIZE}/car-notebook-in.py",
f"{CONTAINERIZE}/car-notebook-in.ipynb",
f"{CONTAINERIZE}/car-multi-notebook-in.ipynb"
]:
content = readfile(file)
content = content.replace("TOKEN", token)
kubeman.hline()
print(content)
kubeman.hline()
out = file.replace("-in", "")
writefile(out, content)
kubeman.banner(out)
destination = out.replace(f"{CONTAINERIZE}/", "")
kubeman.execute("sync")
kubeman.execute(f"cat {out}")
kubeman.execute(f'minikube ssh "sudo chmod -R 777 /nfs"')
kubeman.execute(
f"minikube cp {out} /nfs/indycar/notebooks/{destination}")
kubeman.execute(f'minikube ssh "sudo chmod -R 777 /nfs"')
kubeman.execute(
"minikube cp containerize/IndyCar-API.ipynb /nfs/indycar/notebooks/IndyCar-API.ipynb"
)
kubeman.execute(f'minikube ssh "sudo chmod -R 777 /nfs"')
def read(self, filename=None):
"""
Read a pre-existing cmdline.txt and store it
"""
if filename is None:
raise Exception("read called with no filename")
self.cmdline = readfile(filename).strip()
def gather(user,
names,
source,
destination="~/.ssh/authorized_keys",
dryrun=False,
append=False,
tmp=None):
if type(names) != list:
_names = Parameter.expand(names)
tmp_dir = tmp or path_expand("~/.cloudmesh/tmp")
Shell.mkdir(tmp_dir)
destinations = []
for name in _names:
destinations.append("{name}:{file}")
directory = os.path.dirname(destination)
Shell.mkdir(directory)
if not append:
Shell.rm(path_expand(destination))
writefile(destination, "")
for name in _names:
source = f"{user}@{name}:{destination}"
print(f"{source} -> {tmp_dir}/{destination}")
result = Host.scp(source, f"{tmp_dir}/{destination}-{name}",
dryrun)
with open(path_expand(destination), 'a') as file:
for filename in glob(tmp_dir):
content = readfile(filename)
file.write(content)
def info(cls):
try:
info = readfile('~/.cloudmesh/bridge/info').split('\n')
info = info[:info.index(cls.lease_bookmark) + 1]
except:
Console.error("Cannot execute info command. Has the bridge been made yet?")
sys.exit(1)
try:
curr_leases = sudo_readfile('/var/lib/misc/dnsmasq.leases')
# If cur_leases is not empty, then the first element of each row is the epoch time of the lease expiration date
for i in range(len(curr_leases)):
curr_leases[i] = curr_leases[i].split()
curr_leases[i][0] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(curr_leases[i][0])))
curr_leases[i] = ' '.join(curr_leases[i])
curr_leases = '\n' + '\n'.join(curr_leases)
except:
Console.warning("dnsmasq.leases file not found. No devices have been connected yet")
curr_leases = "\n"
toWrite = '\n'.join(info) + curr_leases
sudo_writefile('~/.cloudmesh/bridge/info', toWrite)
banner(toWrite, color='CYAN')
def test_key_gen(self):
HEADING()
inv_file = '~/.cloudmesh/config_test.yaml'
inv = Inventory(inv_file)
inv.add_directory(host='test_host1',
keyfile='~/.ssh/id_rsa.pub',
service='manager')
inv.save()
c = Configure(inventory=inv_file)
priv_key, pub_key = c.generate_ssh_key(hostname='test_host1')
u = c.build_user_data(name='test_host1')
keys = readfile(filename='~/.ssh/id_rsa.pub').strip().split('\n')
t1 = Userdata()\
.with_authorized_keys(keys=keys)\
.with_ssh_password_login(ssh_pwauth=False)\
.with_locale()\
.with_net_tools()\
.with_hostname(hostname='test_host1')\
.with_hosts(hosts=['127.0.0.1:test_host1'])\
.with_packages(packages='avahi-daemon')\
.with_runcmd(cmd=f'cat /boot/firmware/id_rsa.pub > /home/ubuntu/.ssh/id_rsa.pub')\
.with_runcmd(cmd=f'cat /boot/firmware/id_rsa > /home/ubuntu/.ssh/id_rsa')\
.with_fix_user_dir_owner(user='******')\
.with_runcmd(cmd=f'chmod 600 /home/ubuntu/.ssh/id_rsa')\
.with_runcmd(cmd=f'sudo rm /boot/firmware/id_rsa.pub')\
.with_runcmd(cmd=f'sudo rm /boot/firmware/id_rsa')
assert (t1.content == u.content)
os.system("rm -f " + path_expand(inv_file))
def systeminfo():
data = {
'machine': platform.machine(),
'version': platform.version(),
'platform': platform.platform(),
'system': platform.system(),
'processors': platform.system(),
'sys': sys.platform,
'mac_version': "",
'win_version': "",
'python': sys.version
}
try:
data['node'] = platform.uname().node,
except:
data['node'] = 'unkown'
try:
data['release'] = platform.uname().release,
except:
data['release'] = ''
try:
data['machine'] = platform.uname().machine,
except:
data['machine'] = ''
try:
data['processor'] = platform.uname().processor,
except:
data['processor'] = ''
try:
data['user']= os.environ['USER']
except:
data['user']= ''
try:
data['mac_version'] = platform.mac_ver()[0]
if data['mac_version'] == ('', '', '', ''):
data['mac_version'] = ""
except:
pass
try:
data['win_version'] = platform.win32_ver()
if data['win_version'] == ('', '', '', ''):
data['win_version'] = ""
except:
pass
try:
release_files = Path("/etc").glob("*release")
for filename in release_files:
content = readfile(filename.resolve()).split("\n")
for line in content:
if "=" in line:
attribute, value = line.split("=", 1)
attribute = attribute.replace(" ", "")
data[attribute] = value
except:
pass
return dict(data)
def read(self, filename):
"""
Read a file from the configuration directory
:param filename: The filename
:return: The content
"""
return readfile(filename)
def load(filename=None):
global data
filename = filename or "~/.cloudmesh/inventory.yaml"
if os.path.exists(filename):
content = readfile(filename)
else:
content = sample
data = yaml.safe_load(content)
def read_version_cache(self):
"""
reads the image list cache from the default cache location
:return:
:rtype:
"""
data = yaml.load(readfile(self.cache), Loader=yaml.SafeLoader)
return data
def creae_index_js():
port = socketserver_port()
ip = kubeman.get_minikube_ip()
content = readfile(f"{DASHBOARD}/src/index-in.js")
content = content.replace("MINIKUBEIP",
ip).replace("SOCKETSERVERPORT", port)
writefile(f"{DASHBOARD}/src/index.js", content)
kubeman.execute("sync", driver=os.system)
kubeman.execute(f"cat {DASHBOARD}/src/index.js", driver=os.system)
def systeminfo():
uname = platform.uname()
mem = psutil.virtual_memory()
def add_binary(value):
try:
r = humanize.naturalsize(value, binary=True)
except:
r = ""
return r
data = OrderedDict({
'uname.system': uname.system,
'uname.node': uname.node,
'uname.release': uname.release,
'uname.version': uname.version,
'uname.machine': uname.machine,
'uname.system': uname.system,
'uname.processor': uname.processor,
'sys.platform': sys.platform,
'python': sys.version,
'python.version': sys.version.split(" ", 1)[0],
'python.pip': pip.__version__,
'user': sys_user(),
'mem.percent': str(mem.percent) + " %",
})
for attribute in [
"total", "available", "used", "free", "active", "inactive", "wired"
]:
try:
data[f"mem.{attribute}"] = \
humanize.naturalsize(getattr(mem, attribute), binary=True)
except:
pass
# svmem(total=17179869184, available=6552825856, percent=61.9,
if data['sys.platform'] == 'darwin':
data['platform.version'] = platform.mac_ver()[0]
elif data['sys.platform'] == 'win32':
data['platform.version'] = platform.win32_ver()
else:
data['platform.version'] = uname.version
try:
release_files = Path("/etc").glob("*release")
for filename in release_files:
content = readfile(filename.resolve()).splitlines()
for line in content:
if "=" in line:
attribute, value = line.split("=", 1)
attribute = attribute.replace(" ", "")
data[attribute] = value
except:
pass
return dict(data)
def add(self, name=None, source=None, group=None):
"""
key add [NAME] [--source=FILENAME]
key add [NAME] [--source=git]
key add [NAME] [--source=ssh]
"""
keys = None
if source == "git":
config = Config()
username = config["cloudmesh.profile.github"]
keys = SSHkey().get_from_git(username)
for key in keys:
key['group'] = group or ["git"]
elif source == "ssh":
key = SSHkey(name=name)
key['group'] = group or ["local", "ssh"]
keys = [key]
else:
# source is filename
key = SSHkey()
if not group:
group = os.path.basename(source)
if "." in group:
group = [group.rsplit(".", 1)[0]]
filename = path_expand(source)
print(group)
print(filename)
lines = readfile(filename).splitlines()
pprint(lines)
keys = []
for line in lines:
key = SSHkey()
key.add(key=line, group=group, filename=source)
key["cm"]["name"] = key["name"] = line.split(' ', 2)[2]
keys.append(key)
pprint(keys)
# key = SSHkey(name=name, path=path_expand(source))
# key['group'] = group or ["local", "ssh"]
# keys = [key]
for key in keys:
print(key['name'], key['fingerprint'])
return keys
def shutdown_os(self, name):
Console.ok(f"shutting down server {name}")
# TODO: reading pid from file in current dir for now.
# The pid should be stored in registry longterm.
pid = readfile(f"./{name}_server.pid").strip()
Shell.kill_pid(pid)
def create_metadata(metadata, location):
location = path_expand(location)
Path(os.path.dirname(location)).mkdir(parents=True, exist_ok=True)
if not os.path.isfile(location):
metadata_file = pkg_resources.resource_filename(
"bookmanager",
'template/epub/metadata.txt')
content = readfile(metadata_file)
content = content.format(**metadata)
writefile(location, content)
def run (command):
"""
run the command, redirect the outout to a file and display the content once
it is completed.
:param command:
:return:
"""
os.system (f"{command} &> ./cmd-output")
content = readfile("./cmd-output")
return content
def os_is_pi():
"""
Checks if the os is Raspberry OS
:return: True is Raspberry OS
:rtype: bool
"""
try:
content = readfile('/etc/os-release')
return platform.system() == "Linux" and "raspbian" in content
except: # noqa: E722
return False
def issue(self, repos=None, title=None, file=None):
pprint(repos)
for repo in repos:
if file is not None:
content = readfile(file).strip()
if title is None:
title = content.splitlines()[0]
title = title.replace("#", "").strip()
repository_obj = self.org.get_repo(repo)
repository_obj.create_issue(title=title, body=content)
def distribution(cls):
"""
executes lsb_release command
:param args:
:return:
TODO: needs testing
"""
machine = platform.lower()
result = {"platform": machine,
"distribution": None}
if machine == "linux":
try:
release = readfile("/etc/os-release")
for line in release.splitlines():
attribute, value = line.split("=", 1)
result[attribute] = value
if "Debian" in result["NAME"]:
result["distribution"] = "debian"
elif "Ubuntu" in result["NAME"]:
result["distribution"] = "ubuntu"
except:
try:
r = cls.lsb_release()
for line in r.split():
if ":" in line:
attribute, value = line.split(":", 1)
attribute = attribute.strip().replace(" ",
"_").lower()
value = value.strip()
result[attribute] = value
result["distribution"] = result["description"].split(" ")[
0].lower()
except:
Console.error(
f"lsb_release not found for the platform {machine}")
raise NotImplementedError
elif machine == 'darwin':
result["distribution"] = "macos"
result["version"] = os_platform.mac_ver()[0]
elif machine == 'win32':
result["distribution"] = "windows"
result["version"] = os_platform.win_ver()[0]
else:
Console.error(f"not implemented for the platform {machine}")
raise NotImplementedError
return result
def test_version(self):
HEADING()
version = readfile("VERSION")
Benchmark.Start()
result = Shell.execute("cmsd --version", shell=True)
Benchmark.Stop()
VERBOSE(result)
msg = f"cmsd: {version}"
print("Version string to be tested:", msg)
assert msg in result
Benchmark.Status(True)
def add_to_file(filename, line, warning=None):
"""
adds a line to a file if it is not already in it
:return:
"""
lines = readfile(filename)
if line in lines:
Console.warning(warning)
return
else:
lines += f"\n{line}\n"
writefile(filename, lines)
def test_get(self):
HEADING()
home = self.sourcedir
cloud_path = "/a1.txt"
local_path = f"{home}/hello.txt"
StopWatch.start("GET file")
test_file = self.p.get(cloud_path, local_path, False)
StopWatch.stop("GET file")
assert test_file is not None
content = readfile(local_path)
assert "a1.txt" in content
def get_file_from_local(url, directory, filename):
d = Path(directory)
d.mkdir(parents=True, exist_ok=True)
output = Path(directory) / filename
source = Path(url.replace("file://", "")).resolve()
copyfile(source, output)
r = Result()
r.content = readfile(output).encode()
r.status_code = "200"
return r
def cat_bibfiles(directory, output):
d = path_expand(directory)
bibs = list(Path(d).glob("**/*.bib"))
pprint(bibs)
r = ""
for bib in bibs:
bib = str(bib)
content = readfile(bib)
r = r + "\n\n% " + bib + "\n\n" + content
writefile(output, r)
return list(bibs)
def create_metadata(metadata, location):
location = path_expand(location)
Path(os.path.dirname(location)).mkdir(parents=True, exist_ok=True)
if not os.path.isfile(location):
metadata_file = pkg_resources.resource_filename(
"bookmanager", 'template/epub/metadata.txt')
meta = copy.deepcopy(metadata)
for field in ["author", "title"]:
meta[field] = meta[field].replace("\n", " ")
content = readfile(metadata_file)
content = content.format(**meta)
writefile(location, content)
def add(self, name=None, source=None, group=None):
"""
key add [NAME] [--source=FILENAME]
key add [NAME] [--source=git]
key add [NAME] [--source=ssh]
"""
if source == "git":
config = Config()
username = config["cloudmesh.profile.github"]
keys = SSHkey().get_from_git(username)
for key in keys:
key['group'] = group or ["git"]
elif source == "ssh":
key = SSHkey(name=name)
key['group'] = list(set([group] + ["local", "ssh"]))
keys = [key]
else:
# source is filename
def get_group_name(x):
if x and "." in x:
x = [x.rsplit(".", 1)[0]]
return x
if not group:
group = ["local"]
file_group = os.path.basename(source)
file_group = get_group_name(file_group)
group = group + file_group
filename = path_expand(source)
lines = readfile(filename).splitlines()
keys = []
group = [get_group_name(x) for x in group]
for line in lines:
key = SSHkey()
key_group = group + [line.split("-", 1)[0]]
key.add(key=line, group=key_group, filename=source)
key["cm"]["name"] = key["name"] = line.split(' ', 2)[2]
keys.append(key)
return keys
def gather_keys(
username=None,
hosts=None,
filename="~/.ssh/id_rsa.pub",
key="~/.ssh/id_rsa",
processors=3,
dryrun=False):
"""
returns in a list the keys of the specified hosts
:param username:
:param hosts:
:param filename:
:param key:
:param dryrun:
:return:
"""
names = Parameter.expand(hosts)
results_key = Host.ssh(hosts=names,
command='cat .ssh/id_rsa.pub',
username=username,
verbose=False)
#results_authorized = Host.ssh(hosts=names,
# command='cat .ssh/id_rsa.pub',
# username=username,
# verbose=False)
filename = path_expand(filename)
localkey = {
'host': "localhost",
'command': [''],
'execute': "",
'stdout': readfile(filename).strip(),
'stderr': None,
'returncode': True,
'success': True,
'date': DateTime.now()
}
if results_key is None: # and results_authorized is None:
return ""
# geting the output and also removing duplicates
output = [localkey['stdout']] + \
list(set([element["stdout"] for element in results_key]))
output = '\n'.join(output) + "\n"
return output
def test_put_and_get(self):
HEADING()
home = self.sourcedir
StopWatch.start("PUT file")
test_file = self.p.put(self.p.service, f"{home}/a/a1.txt", "/")
StopWatch.stop("PUT file")
assert test_file is not None
StopWatch.start("GET file")
test_file = self.p.get(self.p.service, f"/a1.txt", f"{home}/hello.txt")
StopWatch.stop("GET file")
assert test_file is not None
content = readfile(f"{home}/hello.txt")
assert "a1.txt" in content
