def backup(self, device=None, to_file=None, blocksize="4m"):
if device is None:
Console.error("Device must have a value")
if to_file is None:
Console.error("To file must have a value")
else:
Sudo.password()
to_file = path_expand(to_file)
size = SDCard.size(device)
to_file = path_expand(to_file)
#
# speed up burning on MacOS
#
if device.startswith("/dev/disk"):
device = device.replace("/dev/disk", "/dev/rdisk")
command = f"sudo dd if={device} bs={blocksize} |" \
f' tqdm --bytes --total {size} --ncols 80|' \
f"dd of={to_file} bs={blocksize}"
print()
Console.info(command)
print()
os.system(command)
def workflow(steps=None):
print(HOME)
print(CONTAINERIZE)
print(STREAMING)
print(DATA)
Sudo.password()
steps = steps or all_steps
try:
for step in steps:
_continue(step.__name__)
step()
StopWatch.benchmark(sysinfo=True,
attributes="short",
csv=False,
total=True)
except Exception as e:
print(e)
StopWatch.benchmark(sysinfo=False,
attributes="short",
csv=False,
total=True)
def launch(file=None):
if file is not None:
if not str(file).endswith(".img"):
raise ValueError(f"file {file} does not end with .img")
if not os.path.exists(file):
raise ValueError(f"image file {file} does not exist")
elif file is None:
file = ""
Imager.install()
if os_is_linux() or os_is_pi():
Sudo.password()
os.system(f"sudo rpi-imager {file}")
elif os_is_mac():
os.system(
f"/Applications/Raspberry\ Pi\ Imager.app/Contents/MacOS/rpi-imager {file} " # noqa: W605
"> /dev/null 2>&1") # noqa: W605
elif os_is_windows():
os.chdir(r"C:\Program Files (x86)\Raspberry Pi Imager")
os.system("rpi-imager.exe")
def install(force=False):
if os_is_mac():
return
if not Imager.installed() or force:
if os_is_linux() or os_is_pi():
Sudo.password()
os.system("sudo apt uninstall -y rpi-imager")
else:
Console.warning("Installation is not supported")
def unmount(self, device=None, card_os="raspberry", full=False):
"""
Unmounts the current SD card. param full indicates whether to use -t flag
:param device: device to unmount, e.g. /dev/sda
:type device: str
:param card_os:
:type card_os:
:param full:
:type full:
:return:
:rtype:
"""
# self.card_os = card_os
os.system('sudo sync') # flush any pending/in-process writes
os.system("sync")
if os_is_linux() or os_is_pi():
Sudo.password()
if full:
_execute(f"eject {device}", f"sudo eject {device}")
else:
# _execute(f"eject {device}", f"sudo eject -t {device}")
device_basename = os.path.basename(device)
result = Shell.run('lsblk')
if device_basename in result.split():
for line in result.splitlines():
line = line.split()
if device_basename in line[0] and len(line) > 6:
Console.ok(f'sudo umount {line[6]}')
os.system(f'sudo umount {line[6]}')
# _execute(f"unmounting {self.boot_volume}", f"sudo umount {self.boot_volume}")
# _execute(f"unmounting {self.root_volume}", f"sudo umount {self.root_volume}")
elif os_is_mac():
_execute(f"unmounting {self.boot_volume}",
f"diskutil umountDisk {device}")
else:
Console.error("Not yet implemented for your OS")
return ""
os.system("sync")
# rm = [f"sudo rmdir {self.boot_volume}",
# f"sudo rmdir {self.root_volume}"]
# for command in rm:
# _execute(command, command)
return True
def fdisk(dev):
"""
calls fdisk on the specified device
:param dev: device, example /dev/sdz
:type dev: str
:return: the output from the fdisk command
:rtype: str
"""
if os_is_mac():
raise NotImplementedError("fdisk -l not supported on MacOS")
else:
Sudo.password()
return subprocess.getoutput(f"sudo fdisk -l {dev}")
def load_device(self, device='dev/sdX'):
"""
Loads the USB device via trayload
:param device: The device on which we format
:type device: str
"""
if os_is_linux() or os_is_pi():
banner(f"load {device}")
Sudo.password()
os.system(f"sudo eject -t {device}")
else:
raise Console.error("Not implemented for this OS")
def writefile(filename=None, content=None, append=False):
"""
Writes the content in the the given file.
:param filename: the filename
:type filename: str
:param content: the content
:type content: str
:param append: if true it append it at the end, otherwise the file will
be overwritten
:type append: bool
:return: the output created by the write process
:rtype: int
"""
os.system("sync")
if append:
content = Sudo.readfile(filename, split=False,
decode=True) + content
command = f"echo '{content}' | sudo cp /dev/stdin {filename}"
if os_is_mac() and "\0" in command:
command = command.replace("\0", "")
os.system(command)
os.system("sync")
return content
def set(ssid=None,
password=None,
country="US",
psk=True,
location=location,
sudo=False):
"""
Sets the wifi. Only works for psk based wifi
:param ssid: The ssid
:type ssid: str
:param password: The password
:type password: str
:param country: Two digit country code
:type country: str
:param psk: If true uses psk authentication
:type psk: bool
:param location: The file where the configuration file should be written to
:type location: str
:param sudo: If tru the write will be done with sudo
:type sudo: bool
:return: True if success
:rtype: bool
"""
if ssid is None or (psk and password is None):
Console.error("SSID or password not set")
return False
if psk:
config = Wifi.template.format(**locals())
else:
config = Wifi.template_key.format(**locals())
try:
if sudo:
Sudo.writefile(location, config)
else:
writefile(location, config)
except FileNotFoundError as e: # noqa: F841
Console.error(f"The file does not exist: {location}")
return False
return True
def execute(command=None, decode="True", debug=False):
"""
Executes the command
:param command: The command to run
:type command: list or str
:param decode:
:type decode:
:param debug:
:type debug:
:return:
:rtype:
"""
result = Sudo.execute(command, decode=decode, debug=debug)
return result
def _readfile(filename=None, split=False, trim=False, decode=True):
"""
Reads the content of the file as sudo and returns the result
:param filename: the filename
:type filename: str
:param split: if true returns a list of lines
:type split: bool
:param trim: trim trailing whitespace. This is useful to
prevent empty string entries when splitting by '\n'
:type trim: bool
:param decode:
:type decode: bool
:return: the content
:rtype: str or list
"""
Sudo.execute("sync")
if os_is_mac():
if decode:
mode = "r"
else:
mode = "rb"
content = common_readfile(filename, mode=mode)
else:
Sudo.password()
result = Sudo.execute(f"cat {filename}", decode=decode)
content = result.stdout
if trim:
content = content.rstrip()
if split:
content = content.splitlines()
Sudo.execute("sync")
return content
if step or run:
if steps.lower() in ["all", "a"]:
kubeman.banner("ALL STEPS")
workflow(steps=all_steps)
elif steps.lower() in ["j", "n", "jupyter", "notebook"]:
kubeman.banner("NOTEBOOK STEPS")
workflow(steps=notebook_steps)
else:
Console.error(f'arguments["WORKFLOW"] does not exist')
elif dashboard:
kubeman.open_k8_dashboard()
elif stormui:
open_storm_ui()
elif clean:
kill_indy_services()
elif info:
deploy_info()
elif arguments["--menu"]:
Sudo.password()
dashboard = True
stormui = True
kubeman.menu(all_steps)
elif arguments["--token"]:
kubeman.get_token()
elif arguments["--mqtt"]:
open_mqtt()
elif arguments["--about"]:
print(LICENSE)
else:
Console.error("Usage issue")
def run(self):
Sudo.password()
host = None
ips = None
hostnames = None
key = None
event = None
tags = None
kind = None
device = None
while True:
event, values = self.window.read()
if event in ("Cancel", 'cancel', None):
if not self.no_diagram:
rack_file = f"~/.cloudmesh/gui/{self.manager}-rack.png"
net_file = f"~/.cloudmesh/gui/{self.manager}-net.png"
os.remove(path_expand(rack_file))
os.remove(path_expand(net_file))
break
#
# UPDATE OS SELECTION
#
if event.startswith("os"):
for image in values:
if image.startswith("os") and values[image]:
break
self.logger(f"Switch OS to: {image}")
image_manager = image_tags[image]["manager"]
image_worker = image_tags[image]["worker"]
self.window[f'tags-{self.manager}'].update(image_manager)
for worker in self.workers:
self.window[f'tags-{worker}'].update(image_worker)
self.window.Refresh()
if event.startswith("button"):
#
# set imaged string
#
imaged = values['imaged']
if not imaged:
self.imaged_str = "--imaged"
else:
self.imaged_str = ""
#
# handle device, hostnames and ips
#
ips = []
hostnames = []
tags = []
for entry in values:
if str(entry).startswith("name"):
hostnames.append(values[entry])
if str(entry).startswith("ip"):
ips.append(values[entry])
if str(entry).startswith("device-") and values[entry]:
device = "/dev/" + entry.replace("device-", "")
if str(entry).startswith("tag") and values[entry]:
tags.append(values[entry])
key = values['key']
self.hostnames_str = ','.join(hostnames)
self.ips_str = ','.join(ips)
#
# get the ssid
#
self.ssid = values['ssid']
self.wifipassword = values['wifi']
host = event.replace("button-", "")
self.set_button_color(host, 'grey')
if host == self.manager:
kind = "manager"
else:
kind = "worker"
print()
print("Host: ", host)
print("IPs: ", ips)
print("Hostnames", hostnames)
print("Ssid: ", self.ssid)
print("Key: ", key)
print("Event: ", event)
print("Tags: ", tags)
print("Kind: ", kind)
print("Device: ", device)
print("Format ", imaged)
print()
# Call burn function for manager and workers
self.logger(f"Burning {kind} {host}")
# tags = values[f'tags-{host}']
self.window[f'status-{host}'].update(' Burning ')
if not self.no_diagram:
self.update_diagram_colors(self.manager, host, "blue")
self.hostnames_str = ','.join(hostnames)
self.ips_str = ','.join(ips)
# command = f"cms burn cluster --device={device}" \
# f" --hostname={self.hostnames_str}" \
# f" --ssid={self.ssid}" \
# f" --wifipassword={self.wifipassword}" \
# f" --ip={self.ips_str}" \
# f" --burning={host}" \
# " -y" \
# f" {self.imaged_str}"
manager, workers = Host.get_hostnames(hostnames)
filename = path_expand(f"~/.cloudmesh/inventory-{manager}.yaml")
Inventory.build_default_inventory(filename=filename, manager=manager,
workers=workers, ips=ips,
gui_images=tags)
if "ubuntu" in tags[0]:
os_cmd = 'ubuntu'
else:
os_cmd = 'raspberry'
if host == manager:
command = f"cms burn {os_cmd} {host}" \
f" --device={device}" \
f" --ssid={self.ssid}" \
f" --wifipassword={self.wifipassword}" \
f" --country={Shell.locale().upper()}"
else:
command = f"cms burn {os_cmd} {host}" \
f" --device={device}"
print(command)
try:
self.logger(f"Executing: {command}")
if self.dryrun:
time.sleep(0.5)
else:
os.system(command)
self.window[f'status-{host}'].update(' Completed ')
if not self.no_diagram:
self.update_diagram_colors(self.manager, host, "green")
self.set_button_color(host, 'green')
except Exception as e:
print(e)
self.logger("Command failed")
if not self.no_diagram:
self.update_diagram_colors(self.manager, host, "orange")
self.set_button_color(host, 'red')
self.window.FindElement(f'button-{host}').Update(button_color=('white', 'green'))
self.window.Refresh()
print('exit')
self.window.close()
def do_host(self, args, arguments):
"""
::
Usage:
host scp NAMES SOURCE DESTINATION [--dryrun]
host ssh NAMES COMMAND [--dryrun] [--output=FORMAT]
host config NAMES --ips=IPS [--user=USER] [--key=PUBLIC]
host config --proxy=PROXY NAMES [--user=USER] [--append] [--local=no] [--StrictHostKeyChecking=no] [--cluster=name]
host config NAMES [--user=USER] [--append] [--local=no] [--StrictHostKeyChecking=no] [--cluster=name]
host find [NAMES] [--user=USER] [--table|--json] [--verbose]
host check NAMES [--user=USER] [--key=PUBLIC]
host key create NAMES [--user=USER] [--dryrun] [--output=FORMAT]
host key list NAMES [--output=FORMAT]
host key setup NAMES
host key gather NAMES [--authorized_keys] [FILE]
host key scatter NAMES [FILE] [--user=USER]
host key add NAMES [FILE]
host key delete NAMES [FILE]
host key access NAMES [FILE] [--user=USER]
host tunnel create NAMES [--port=PORT]
host mac NAMES [--eth] [--wlan] [--output=FORMAT]
host setup WORKERS [LAPTOP]
host shutdown NAMES
host reboot NAMES
host adduser NAMES USER
host passwd NAMES USER
host addsudo NAMES USER
host deluser NAMES USER
host ping NAMES
host info NAMES
This command does some useful things.
Arguments:
FILE a file name
Options:
--dryrun shows what would be done but does not execute
--output=FORMAT the format of the output
--port=PORT starting local port for tunnel assignment
--local=no do not append .local to manager hostname [default: yes]
--user=USER username for manager and workers [default: pi]
--ips=IPS ip addresses of the manager and workers
--StrictHostKeyChecking=no if set to yes, strict host checking is enforced [default: no]
--ProxyJump=no if set to yes, a proxyjump is performed for each worker through the manager [default: yes]
Description:
host scp NAMES SOURCE DESTINATION
Uses scp to transfer Source to NAMES:DESTINATION.
host ssh NAMES COMMAND
runs the command on all specified hosts
Example:
ssh red[01-10] \"uname -a\"
host key create NAMES
create a ~/.ssh/id_rsa and id_rsa.pub on all hosts specified
Example:
ssh key create "red[01-10]"
host key list NAMES
list all id_rsa.pub keys from all hosts specifed
Example:
ssh key list red[01-10]
host key gather HOSTS FILE
gathers all keys from file FILE including the one from localhost.
ssh key gather "red[01-10]" keys.txt
host key scatter HOSTS FILE [--user=USER]
copies all keys from file FILE to authorized_keys on all hosts,
but also makes sure that the users ~/.ssh/id_rsa.pub key is in
the file. If provided the optional user, it will add the keys to
that user's .ssh directory. This is often required when
adding a new user in which case HOSTS should still a sudo
user with ssh currently enabled.
1) adds ~/.id_rsa.pub to the FILE only if its not already in it
2) removes all duplicated keys
Example:
ssh key scatter "red[01-10]"
ssh key scatter pi@red[01-10] keys.txt --user=alice
host key add NAMES FILE
Adds all keys in FILE into the authorized_keys of NAMES.
Example:
cms host key add worker001 ~/.ssh/id_rsa.pub
host key delete NAMES FILE
Deletes all keys in fILE from authorized_keys of NAMES if they exist.
Example
cms host key delete worker001 ~/.ssh/id_rsa.pub
host key scp NAMES FILE
copies all keys from file FILE to authorized_keys on all hosts
but also makes sure that the users ~/.ssh/id_rsa.pub key is in
the file and removes duplicates, e.g. it calls fix before upload
Example:
ssh key list red[01-10] > pubkeys.txt
ssh key scp red[01-10] pubkeys.txt
host config NAMES IPS [--user=USER] [--key=PUBLIC]
generates an ssh config file tempalte that can be added to your
.ssh/config file
Example:
cms host config "red,red[01-03]" "198.168.1.[1-4]" --user=pi
host check NAMES [--user=USER] [--key=PUBLIC]
This command is used to test if you can login to the specified
hosts. It executes the hostname command and compares it.
It provides a table with a sucess column
cms host check "red,red[01-03]"
+-------+---------+--------+
| host | success | stdout |
+-------+---------+--------+
| red | True | red |
| red01 | True | red01 |
| red02 | True | red02 |
| red03 | True | red03 |
+-------+---------+--------+
host tunnel create NAMES [--port=PORT]
This command is used to create a persistent local port
forward on the host to permit ssh tunnelling from the wlan to
the physical network (eth). This registers an autossh service in
systemd with the defualt port starting at 8001.
Example:
cms host tunnel create red00[1-3]
host mac NAMES
returns the list of mac addresses of the named pis.
host setup WORKERS [LAPTOP]
Executes the following steps
cms bridge create --interface='wlan0'
cms host key create red00[1-3]
cms host key gather red00[1-3],[email protected] keys.txt
cms host key scatter red00[1-3],localhost keys.txt
rm keys.txt
cms host tunnel create red00[1-3]
host shutdown NAMES
Shutsdown NAMES with `sudo shutdown -h now`. If localhost in
names, it is shutdown last.
host reboot NAMES
Reboots NAMES with `sudo reboot`. If localhost in names,
it is rebooted last.
host adduser NAMES USER
Adds a user with user name USER to the hosts identified by
NAMES. Password is disabled, see host passwd to enable.
host addsudo NAMES USER
Adds sudo rights to USER at NAMES
host passwd NAMES USER
Changes the password for USER at NAMES
host deluser NAMES USER
Deleted USER from NAMES. Home directory will be removed.
host config proxy PROXY NAMES
This adds to your ~/.ssh/config file a ProxyJump
configuration to reach NAMES via PROXY. This is useful when
the PROXY is acting as a network bridge for NAMES to your
current device.
Example:
cms host config proxy [email protected] red00[1-2]
"""
def _print(results):
arguments.output = arguments.output or 'table'
if arguments.output in ['table', 'yaml']:
print(
Printer.write(
results,
order=['host', 'success', 'stdout', 'stderr'],
output=arguments.output))
else:
pprint(results)
def _print_pis(results):
arguments.output = arguments.output or 'table'
if arguments.output in ['table', 'yaml']:
print(
Printer.write(results,
order=[
'name', 'ip', 'user', 'os', 'mac',
'model', 'memory', 'serial', ".local"
],
output=arguments.output))
# not printed "revision"
# not printed "hardware"
else:
pprint(results)
def get_filename(filename, hosts):
if filename is not None:
return filename
if type(hosts) == str:
hosts = Parameter.expand(hosts)
label = hosts[0]
return path_expand(f"~/.ssh/cluster_keys_{label}")
map_parameters(arguments, 'eth', 'wlan'
'dryrun', 'output', 'user', 'port', 'append',
'StrictHostKeyChecking', 'local', 'proxy', 'ips',
'cluster')
dryrun = arguments.dryrun
# VERBOSE(arguments)
if dryrun:
VERBOSE(arguments)
if arguments.info:
names = Parameter.expand(arguments.names)
# check if .local
# check if mesh network
# check if static network
# use arp - a di find hosts ips
# if linux
# dig +short -x 192.168.50.1
Console.error("Not yet Implemented")
elif arguments.find:
verbose = arguments["--verbose"]
names = Parameter.expand(arguments.NAMES)
# temporary so we can easy modify while not needing to update cloudmesh.common
from cloudmesh.host.network import PiNetwork
network = PiNetwork()
pis = network.find_pis(user=arguments.user, verbose=verbose)
if arguments["--json"]:
print(pis)
else:
_print_pis(pis)
elif arguments.mac:
names = Parameter.expand(arguments.NAMES)
if not arguments.eth and not arguments.wlan:
arguments.eth = True
arguments.wlan = True
eth = 'cat /sys/class/net/eth0/address'
wlan = 'cat /sys/class/net/wlan0/address'
if arguments.eth:
results = Host.ssh(hosts=names,
command=eth,
username=arguments.user)
print("eth0:")
_print(results)
if arguments.wlan:
results = Host.ssh(hosts=names,
command=wlan,
username=arguments.user)
print("wlan0:")
_print(results)
elif arguments.setup:
HostCreate.setup(workers=arguments.WORKERS,
laptop=arguments.LAPTOP)
elif arguments.scp and not arguments.key:
result = Host.put(hosts=arguments.NAMES,
source=arguments.SOURCE,
destination=arguments.DESTINATION)
_print(result)
elif arguments.ping:
names = Parameter.expand(arguments.NAMES)
# print (names)
results = Host.ping(hosts=names)
_print(results)
elif arguments.ssh:
names = Parameter.expand(arguments.NAMES)
# print (names)
results = Host.ssh(hosts=names, command=arguments.COMMAND)
_print(results)
elif arguments.key and arguments.create:
results = Host.ssh_keygen(hosts=arguments.NAMES,
username=arguments.user,
dryrun=dryrun)
_print(results)
elif arguments.key and arguments.list:
names = Parameter.expand(arguments.NAMES)
results = Host.ssh(hosts=names,
command='cat .ssh/id_rsa.pub',
username=arguments.user)
_print(results)
elif arguments.key and arguments.add:
filename = get_filename(arguments.NAMES)
if not os.path.isfile(filename):
Console.error(f"Cannot find file {filename}")
return
# Copy to temp location
Host.put(hosts=arguments.NAMES,
source=filename,
destination="~/.ssh/key.tmp")
# Execute append command and remove command
command = 'cat ~/.ssh/key.tmp >> ~/.ssh/authorized_keys && rm ~/.ssh/key.tmp'
Host.ssh(hosts=arguments.NAMES, command=command)
elif arguments.key and arguments.delete:
Console.ok("key delete")
filename = get_filename(arguments.FILE, arguments.NAMES)
if not os.path.isfile(filename):
Console.error(f"Cannot find file {filename}")
return
# Copy to temp location
remote_temp = "~/.ssh/key.tmp"
Host.put(hosts=arguments.NAMES,
source=filename,
destination=remote_temp)
# grep can read multiple patterns from a file, one per line. Combine with
# the options -v to output non-matching lines, and -F to match strings
# instead of regex and -x to require that the whole line matches.
command = f"""grep -Fvx -f {remote_temp} ~/.ssh/authorized_keys >remaining_keys && \
mv remaining_keys ~/.ssh/authorized_keys && \
rm {remote_temp}"""
Host.ssh(hosts=arguments.NAMES, command=command)
Console.ok(f"Delete keys from {filename} on {arguments.NAMES}")
elif arguments.key and arguments.setup:
label = Parameter.expand(arguments.NAMES)[0]
filename = get_filename(arguments.FILE, arguments.NAMES)
directory = os.path.dirname(filename)
if directory:
Shell.mkdir(directory)
output = Host.gather_keys(username=arguments.user,
hosts=arguments.NAMES,
filename="~/.ssh/id_rsa.pub",
key="~/.ssh/id_rsa",
processors=3,
dryrun=False)
with open(filename, "w") as f:
f.write(output)
# scatter
# place .ssh/config a trict host check to no
elif arguments.key and arguments.gather:
output = Host.gather_keys(username=arguments.user,
hosts=arguments.NAMES,
filename="~/.ssh/id_rsa.pub",
key="~/.ssh/id_rsa",
processors=3,
dryrun=False)
VERBOSE(arguments)
filename = get_filename(arguments.FILE, arguments.NAMES)
print(output)
banner(f"Writing Keys to file {filename}")
directory = os.path.dirname(filename)
print('command directory', directory)
if directory:
Shell.mkdir(directory)
if os.path.isfile(filename) and yn_choice(
f'{filename} is not empty. Do you wish to overwrite it? (If no you will append).'
):
with open(filename, "w") as f:
f.write(output)
else:
with open(filename, "a") as f:
f.write(output)
elif arguments.key and arguments.scatter:
#
# this should be a function in Host
#
filename = get_filename(arguments.FILE, arguments.NAMES)
names = arguments.NAMES
user = arguments.user
if not os.path.isfile(filename):
Console.error("The file does not exist")
return ""
if not user:
result = Host.put(hosts=names,
source=filename,
destination=".ssh/authorized_keys")
_print(result)
else:
Console.info('SCP to ./temp_authorzied_keys_temp')
result = Host.put(hosts=names,
source=filename,
destination="temp_authorized_keys_temp")
_print(result)
Console.info(f'Mkdir /home/{user}/.ssh if not exist')
command = f'sudo mkdir -p /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Chown /home/{user}/.ssh to {user}')
command = f'sudo chown {user}:{user} /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Chmod /home/{user}/.ssh to 700')
command = f'sudo chmod 700 /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Mv temp_authorized_keys_temp to /home/'
f'{user}/.ssh/authorized_keys')
command = f'sudo mv temp_authorized_keys_temp /home/' \
f'{user}/.ssh/authorized_keys'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Chown /home/{user}/.ssh/authorized_keys to '
f'{user}')
command = f'sudo chown {user}:{user} /home/' \
f'{user}/.ssh/authorized_keys'
result = Host.ssh(hosts=names, command=command)
_print(result)
elif arguments.key and arguments.access:
#
# this should be a function in Host
#
names = arguments.NAMES
user = arguments.user
filename = arguments.FILE
temp = path_expand("~/.cloudmesh/temp_config")
if filename:
config = readfile(filename)
else:
config = textwrap.dedent("""
Host *
StrictHostKeyChecking no
""").strip()
writefile(temp, config)
if not os.path.isfile(temp):
Console.error("The file does not exist")
return ""
if not user:
result = Host.put(hosts=names,
source=temp,
destination=".ssh/config")
_print(result)
else:
Console.info(f'Mkdir /home/{user}/.ssh if not exist')
command = f'sudo mkdir -p /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info('SCP to ./temp_config')
result = Host.put(hosts=names,
source=temp,
destination=".ssh/config")
_print(result)
Console.info(f'Chown /home/{user}/.ssh to {user}')
command = f'sudo chown {user}:{user} /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Chmod /home/{user}/.ssh to 700')
command = f'sudo chmod 700 /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
elif arguments.config and arguments.ips:
print("NNNNNNNN")
key = arguments.key or "~/.ssh/id_rsa.pub"
result = Host.config(hosts=arguments.NAMES,
ips=arguments.IPS,
username=arguments.user,
key=key)
print(result)
"""
host config NAMES --ips=IPS [--user=USER] [--key=PUBLIC]
host config --proxy=PROXY NAMES [--user=USER] [--append] [--local=no] [--StrictHostKeyChecking=no]
host config NAMES [--user=USER] [--append] [--local=no] [--StrictHostKeyChecking=no]
"""
elif arguments.config:
if str_bool(arguments.local):
local_str = ".local"
else:
local_str = ""
if str_bool(arguments.StrictHostKeyChecking):
strict_host_str = "yes"
else:
strict_host_str = "no"
names = Parameter.expand(arguments.NAMES)
user = arguments.user
if arguments.cluster:
cluster = arguments.cluster
else:
# take the first name and remove spaces
cluster = ''.join([i for i in names[0] if not i.isdigit()])
ssh_config_output = ""
ssh_config_output = f'\n##### CLOUDMESH PROXY CONFIG {cluster} #####\n\n'
if arguments.proxy:
proxy_host = arguments.proxy
proxy_jump = f' ProxyJump {proxy_host}\n'
ssh_config_output += \
f'Host {proxy_host}\n' \
f' HostName {proxy_host}{local_str}\n' \
f' User {user}\n' \
f' PreferredAuthentications publickey\n' + \
f' StrictHostKeyChecking {strict_host_str}\n'
ssh_config_output += '\n'
else:
proxy_jump = ""
"""
ssh_config_output = f'\n##### CLOUDMESH PROXY CONFIG {cluster} #####\n\n'\
f'Host {proxy_host}\n' \
f' HostName {proxy_host}{local_str}\n' \
f' User {user}\n' \
f' StrictHostKeyChecking {strict_host_str}\n\n'
"""
### the local_str in the hostname may be wrong since its not manager
for name in names:
ssh_config_template = f'Host {name}\n' \
f' HostName {name}{local_str}\n' \
f' User {user}\n' \
f' PreferredAuthentications publickey\n' + \
f' StrictHostKeyChecking {strict_host_str}\n' + \
proxy_jump
ssh_config_template += '\n'
ssh_config_output += ssh_config_template
ssh_config_output += f'##### CLOUDMESH PROXY CONFIG {cluster} #####\n'
print('Adding to ~/.ssh/config')
print(ssh_config_output)
if not os.path.exists(path_expand('~/.ssh/config')):
with open(path_expand('~/.ssh/config'), 'w+') as f:
f.write(ssh_config_output)
else:
f = open(path_expand('~/.ssh/config'), 'r')
lines = f.readlines()
f.close()
with open(path_expand('~/.ssh/config'), 'w+') as f:
if f'##### CLOUDMESH PROXY CONFIG {cluster} #####\n' in lines:
start = lines.index(
f'##### CLOUDMESH PROXY CONFIG {cluster} #####\n')
lines.reverse()
end = lines.index(
f'##### CLOUDMESH PROXY CONFIG {cluster} #####\n')
end = len(lines) - end - 1
lines.reverse()
original_config = lines[start:end + 1]
del lines[start:end + 1]
f.writelines(lines)
if arguments.append:
f.writelines(original_config)
f.write(ssh_config_output)
else:
f.write(ssh_config_output)
else:
f.writelines(lines)
f.write(ssh_config_output)
f.close()
elif arguments.check:
key = arguments.key or "~/.ssh/id_rsa.pub"
result = Host.check(hosts=arguments.NAMES,
username=arguments.user,
key=key)
for entry in result:
entry['success'] = entry['stdout'] == entry['host']
_print(result)
elif arguments.tunnel and arguments.create:
wlan_ip = Shell.run("hostname -I | awk '{print $2}'").strip()
print(f'\nUsing wlan0 IP = {wlan_ip}')
hostname = Shell.run("hostname").strip()
print(f'Using cluster hostname = {hostname}')
names = Parameter.expand(arguments.NAMES)
port = arguments.port or "8001"
ssh_config_output = f'Host {hostname}\n' \
f' HostName {hostname}.local\n' \
f' User pi\n\n'
for name in names:
service_name = f"autossh-{name}.service"
service_template = "[Unit]\n" \
f"Description=AutoSSH tunnel service to {name} on local port " \
f"{port}\n" \
"After=multi-user.target\n\n" \
"[Service]\n" \
"User=pi\n" \
"Group=pi\n" \
'Environment="AUTOSSH_GATETIME=0"\n' \
'ExecStart=/usr/bin/autossh -M 0 -o "ServerAliveInterval 30" ' \
'-o "ServerAliveCountMax 3" -i ' \
'/home/pi/.ssh/id_rsa -NL ' \
f'{wlan_ip}:{port}:localhost:22 p' \
f'i@{name}\n\n' \
"[Install]\n" \
"WantedBy=multi-user.target"
ssh_config_template = f'Host {name}\n' \
f' HostName {hostname}.local\n' \
f' User pi\n' \
f' Port {port}\n\n'
ssh_config_output += ssh_config_template
Sudo.writefile(f'/etc/systemd/system/{service_name}',
service_template)
port = str(int(port) + 1)
os.system('sudo systemctl daemon-reload')
for name in names:
servicename = f"autossh-{name}.service"
os.system(f'sudo systemctl start {servicename}')
os.system(f'sudo systemctl enable {servicename}')
print('\nTunnels created.\n\nPlease place the following in your '
'remote machine\'s (i.e. laptop) ~/.ssh/config file to '
'alias simple ssh access (i.e. ssh red001).')
banner('copy to ~/.ssh/config on remote host (i.e laptop)')
print(ssh_config_output)
elif arguments.shutdown or arguments.reboot:
if arguments.shutdown:
command = 'sudo shutdown -h now'
elif arguments.reboot:
command = 'sudo reboot'
names = Parameter.expand(arguments.NAMES)
hostname = Shell.run("hostname").strip()
localhost = None
if "localhost" in names:
names.remove("localhost")
localhost = True
if hostname in names:
names.remove(hostname)
localhost = True
manager, workers = Host.get_hostnames(names)
if workers:
Console.info(f'Executing `{command}` for {workers}')
Host.ssh(hosts=workers, command=command)
if manager:
Console.info(f'Executing `{command}` for {manager}')
Host.ssh(hosts=manager, command=command)
#_print(results)
# results can be misleading becuase there is a race between the
# shutdown and the error code being returned from the ssh processes.
if localhost:
os.system(command)
elif arguments.adduser:
names = Parameter.expand(arguments.NAMES)
user = arguments.USER
localhost = None
if 'localhost' in names:
localhost = 'localhost'
elif platform.node() in names:
localhost = platform.node()
if localhost in names:
print('\nAdding user to localhost')
result = Shell.run(f'sudo adduser {user} '
f'--disabled-password '
f'--gecos "" ')
print(result)
names.remove(localhost)
if len(names) > 0:
command = f"sudo adduser {user} --disabled-password --gecos ',' "
results = Host.ssh(hosts=names, command=command)
_print(results)
elif arguments.passwd:
names = Parameter.expand(arguments.NAMES)
user = arguments.USER
localhost = None
if 'localhost' in names:
localhost = 'localhost'
elif platform.node() in names:
localhost = platform.node()
if localhost in names:
print("\nSetting password on localhost, please provide user "
"password")
result = os.system(f'sudo passwd {user}')
print(result)
names.remove(localhost)
if len(names) > 0:
print("\nSetting password on remote hosts, please enter user "
"password\n")
password = getpass("Please enter the user password:"******"{password}\n{password}" | sudo passwd {user}'
results = Host.ssh(hosts=names, command=command)
_print(results)
elif arguments.addsudo:
names = Parameter.expand(arguments.NAMES)
user = arguments.USER
localhost = None
if 'localhost' in names:
localhost = 'localhost'
elif platform.node() in names:
localhost = platform.node()
if localhost in names:
print('\nAdding user to sudo group on localhost')
result = Shell.run(f'sudo adduser {user} sudo')
print(result)
names.remove(localhost)
if len(names) > 0:
command = f'sudo adduser {user} sudo'
results = Host.ssh(hosts=names, command=command)
_print(results)
elif arguments.deluser:
names = Parameter.expand(arguments.NAMES)
user = arguments.USER
if 'localhost' in names:
localhost = 'localhost'
elif platform.node() in names:
localhost = platform.node()
if localhost in names:
print('\nDeleting user on localhost')
result = Shell.run(f'sudo userdel -r {user}')
print(result)
names.remove(localhost)
if len(names) > 0:
command = f'sudo userdel -r {user}'
results = Host.ssh(hosts=names, command=command)
_print(results)
return ""
def burn_sdcard(self,
image=None,
tag=None,
device=None,
blocksize="4M",
name="the inserted card",
yes=False):
"""
Burns the SD Card with an image
:param image: Image object to use for burning (used by copy)
:type image: str
:param name:
:type name: str
:param tag: tag object used for burning (used by sdcard)
:type tag: str
:param device: Device to burn to, e.g. /dev/sda
:type device: str
:param blocksize: the blocksize used when writing, default 4M
:type blocksize: str
:param yes:
:type yes: str
"""
if image and tag:
Console.error("Implementation error, burn_sdcard can't have image "
"and tag.")
return ""
Console.info(f"Burning {name} ...")
if image is not None:
image_path = image
else:
image = Image().find(tag=tag)
if image is None:
Console.error("No matching image found.")
return ""
elif len(image) > 1:
Console.error("Too many images found")
print(
Printer.write(image,
order=["tag", "version"],
header=["Tag", "Version"]))
return ""
image = image[0]
if "ubuntu" in image["url"]:
_name = os.path.basename(Image.get_name(image["url"]))
_name = _name.replace(".xz", "")
else:
_name = os.path.basename(Image.get_name(image["url"])) + ".img"
image_path = Image().directory + "/" + _name
print(image_path)
if not os.path.isfile(image_path):
tags = ' '.join(tag)
print()
Console.error(
f"Image with tags '{tags}' not found. To download use")
print()
Console.msg(f"cms burn image get {tags}")
print()
return ""
orig_size = size = humanize.naturalsize(os.path.getsize(image_path))
# size = details[0]['size']
n, unit = size.split(" ")
unit = unit.replace("GB", "G")
unit = unit.replace("MB", "M")
n = float(n)
if unit == "G":
n = n * 1000**3
elif unit == "M":
n = n * 1000**2
size = int(n)
banner(f"Preparing the SDCard {name}")
print(f"Name: {name}")
print(f"Image: {image_path}")
print(f"Image Size: {orig_size}")
print(f"Device: {device}")
print(f"Blocksize: {blocksize}")
if os_is_mac():
blocksize = blocksize.lower()
print()
Sudo.password()
if device is None:
Console.error("Please specify a device")
return
#
# speedup burn for MacOS
#
if device.startswith("/dev/disk"):
device = device.replace("/dev/disk", "/dev/rdisk")
if os_is_mac():
details = USB.get_from_diskutil()
USB.print_details(details)
if not (yes or yn_choice(f"\nDo you like to write {name} on {device} "
f"with the image {image_path}")):
return ""
# TODO Gregor verify this is ok commenting out this line
# self.mount(device=device)
if os_is_mac():
command = f"sudo dd if={image_path} bs={blocksize} |" \
f' tqdm --bytes --total {size} --ncols 80 |' \
f" sudo dd of={device} bs={blocksize}"
else:
# command = f"sudo dd if={image_path} of={device} bs={blocksize} status=progress conv=fsync"
command = f"sudo dd if={image_path} bs={blocksize} oflag=direct |" \
f' tqdm --bytes --total {size} --ncols 80 |' \
f" sudo dd of={device} bs={blocksize} iflag=fullblock " \
f"oflag=direct conv=fsync"
print(command)
os.system(command)
Sudo.execute("sync")
if os_is_linux():
self.unmount(device=device, full=True)
else:
self.unmount(device=device)
def mount(self, device=None, card_os="raspberry"):
"""
Mounts the current SD card
"""
if os_is_linux():
Sudo.password()
dmesg = USB.get_from_dmesg()
# TODO Need a better way to identify which sd card to use for mounting
# instead of iterating over all of them
for usbcard in dmesg:
dev = device or usbcard['dev']
print(f"Mounting filesystems on {dev}")
try:
Console.ok(f"mounting {device}")
os.system(
'sudo sync') # flush any pending/in-process writes
os.system(f"sudo eject {device}")
os.system(f"sudo eject -t {device}")
os.system(
'sudo sync') # flush any pending/in-process writes
# ensure the card is mounted before returning
device_basename = os.path.basename(device)
part1 = False
part2 = False
for i in range(20):
result = Shell.run('lsblk')
if device_basename in result.split():
for line in result.splitlines():
line = line.split()
if device_basename + '1' in line[0] and len(
line) > 6:
part1 = True
elif device_basename + '2' in line[0] and len(
line) > 6:
part2 = True
if part1 and part2:
# card is fully mounted
break
time.sleep(0.5)
if not part1 and not part2:
raise Exception("card failed to mount both partitions")
except Exception as e:
print(e)
elif os_is_pi():
Sudo.password()
if card_os is None:
Console.error("Please specify the OS you have on the SD Card")
return ""
self.card_os = card_os
dmesg = USB.get_from_dmesg()
print(dmesg)
# TODO Need a better way to identify which sd card to use for mounting
# instead of iterating over all of them
os.system('sudo sync') # flush any pending/in-process writes
for usbcard in dmesg:
dev = device or usbcard['dev']
print(
f"Mounting filesystems on {dev} assuming it is {card_os} as you specified"
)
sd1 = f"{dev}1"
sd2 = f"{dev}2"
try:
if os.path.exists(sd1):
Console.ok(f"mounting {sd1} {self.boot_volume}")
os.system(f"sudo mkdir -p {self.boot_volume}")
os.system(
f"sudo mount -t vfat {sd1} {self.boot_volume}")
except Exception as e:
print(e)
try:
if os.path.exists(sd2):
Console.ok(f"mounting {sd2} {self.root_volume}")
os.system(f"sudo mkdir -p {self.root_volume}")
os.system(
f"sudo mount -t ext4 {sd2} {self.root_volume}")
except Exception as e:
print(e)
elif os_is_mac():
command = f"diskutil mountDisk {device}"
print(command)
os.system(command)
else:
Console.error("Not yet implemented for your OS")
Sudo.execute("sync")
return ""
def format_device(self,
device='dev/sdX',
unmount=True,
yes=False,
verbose=True):
"""
Formats device with one FAT32 partition
WARNING: make sure you have the right device, this command could
potentially erase your OS
:param device: The device on which we format
:type device: str
:param unmount:
:type unmount:
:param yes:
:type yes:
:param verbose:
:type verbose:
:return:
:rtype:
"""
_title = "UNTITLED"
def prepare_sdcard():
"""
ensures a card is detected and unmounted
:return: True if prepared
:rtype: bool
"""
#
Console.ok(f'sudo eject -t {device}')
os.system(f'sudo eject -t {device}')
time.sleep(3)
device_basename = os.path.basename(device)
result = Shell.run('lsblk')
if device_basename in result.split():
for line in result.splitlines():
line = line.split()
if device_basename in line[0] and len(line) > 6:
Console.ok(f'sudo umount {line[6]}')
os.system(f'sudo umount {line[6]}')
return True
else:
Console.error("SD Card not detected. Please reinsert "
"card reader. ")
if not yn_choice("Card reader re-inserted? No to cancel "
"operation"):
return False
else:
time.sleep(3)
return prepare_sdcard()
Sudo.password()
if os_is_linux() or os_is_pi():
if verbose:
banner(f"format {device}")
else:
print(f"format {device}")
if not prepare_sdcard():
return False
# TODO Gregor verify commenting out the below is ok
# if os_is_mac():
# self.mount(device=device)
user = os.environ.get('USER')
script = textwrap.dedent(f"""
ls /media/{user}
sudo parted {device} --script -- mklabel msdos
sudo parted {device} --script -- mkpart primary fat32 1MiB 100%
sudo mkfs.vfat -n {_title} -F32 {device}1
sudo parted {device} --script print""").strip().splitlines()
for line in script:
_execute(line, line)
os.system("sudo sync")
if unmount:
self.unmount(device=device
) # without dev we unmount but do not eject. If
# we completely eject, burn will fail to detect the device.
os.system("sudo sync")
Console.ok("Formatted SD Card")
elif os_is_mac():
details = USB.get_dev_from_diskutil()
# checking if string contains list element
valid = any(entry in device for entry in details)
if not valid:
Console.error(
f"this device can not be used for formatting: {device}")
return False
elif len(details) > 1:
Console.error(
"For security reasons, please only put one USB writer in")
Console.msg(f"we found {details}")
return False
else:
details = USB.get_from_diskutil()
USB.print_details(details)
print()
if yes or yn_choice(
f"\nDo you like to format {device} as {_title}"):
_execute(
f"Formatting {device} as {_title}",
f"sudo diskutil eraseDisk FAT32 {_title} MBRFormat {device}"
)
else:
raise NotImplementedError("Not implemented for this OS")
return True