def Prints(self, data, output='table', kind=None):
if kind is None and len(data) > 0:
kind = data[0]["cm"]["kind"]
if output in ["flat", "table"]:
order = self.p.output[kind]['order'] # not pretty
header = self.p.output[kind]['header'] # not pretty
if 'humanize' in self.p.output[kind]:
humanize = self.p.output[kind]['humanize']
else:
humanize = None
_output = Printer.flatwrite(data,
sort_keys=["name"],
order=order,
header=header,
output=output,
humanize=humanize)
else:
_output = Printer.write(data, output=output)
return _output
def status(self, user=None):
reservation = self.reservations(user=user)
VERBOSE(reservation)
print(
Printer.write(
reservation,
order=[
'Nodes', 'ReservationName', 'State', 'Users', 'CoreCnt',
'Flags', 'NodeCnt'
],
#header=[
# 'Nodes',
# 'Reservation',
# 'State',
# 'Allowed Users'
# 'Cores (CPUs)',
# 'Flags',
# 'No.Nodes'
#]
))
reserved = self.reserved_nodes(user=user)
#
# BUG check for date
#
# print (reserved)
used = {}
for host in ["romeo", "volta"]:
status = self.queue(host=host, user=user)
for key in status:
used_gpu = status[key]
used[key] = {'name': key, 'used': status[key], 'domain': host}
print(
Printer.write(used,
order=["name", "domain", "used"],
header=["Host", "Domain", "Used GPUs"]))
data = {}
for host in ["romeo", "volta"]:
users = self.users(host=host, user=user)
users = ', '.join(users)
data[host] = {"host": host, "users": users}
print("\nActive Users")
print(
Printer.write(data,
order=["host", "users"],
header=["Host", "Users"]))
def Print_leds(self, arguments, results):
if arguments.output == 'table':
print(Printer.write(results,
order=['host', 'green', 'red']))
else:
pprint(results)
def Print(self, arguments, results):
if arguments.output == 'table':
print(Printer.write(results,
order=['host', 'success', 'stdout']))
else:
pprint(results)
def load_data(self):
self.background = '#64778d'
if os_is_mac():
self.details = USB.get_from_diskutil()
else:
self.details = USB.get_from_dmesg()
self.devices = yaml.safe_load(Printer.write(self.details,
order=[
"dev",
"info",
"formatted",
"size",
"active",
"readable",
"empty",
"direct-access",
"removable",
"writeable"],
header=[
"Path",
"Info",
"Formatted",
"Size",
"Plugged-in",
"Readable",
"Empty",
"Access",
"Removable",
"Writeable"],
output="yaml"))
def print_details(details, order=None, header=None, output="table"):
if order is None:
order = ["dev",
"info",
"formatted",
"size",
"active",
"readable",
"empty",
"direct-access",
"removable",
"writeable"
],
if header is None:
header = ["Path",
"Info",
"Formatted",
"Size",
"Plugged-in",
"Readable",
"Empty",
"Access",
"Removable",
"Writeable"
],
# print(Printer.write(details, order=order, header=header, output=output))
print(Printer.write(details, order=order[0], header=header[0]))
def install(self):
"""
Installs /usr/local/bin/pishrink.sh
Installs parted
:return:
:rtype:
"""
if os_is_mac():
Console.error("This command is not supported on MacOS")
return ""
else:
banner("Installing pishrink.sh into /usr/local/bin")
script_name = Shell.download(
'https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh',
'pishrink.sh',
provider='system')
script = \
f"""
chmod +x {script_name}
sudo mv {script_name} /usr/local/bin
"""
result = JobScript.execute(script)
print(
Printer.write(result,
order=[
"name", "command", "status", "stdout",
"returncode"
]))
if os_is_linux() or os_is_pi():
banner("Installing pishrink.sh into /usr/local/bin")
script = \
"""
sudo apt install parted -y > $HOME/tmp.log
"""
result = JobScript.execute(script)
print(
Printer.write(result,
order=[
"name", "command", "status", "stdout",
"returncode"
]))
def smart_login(self, user=None, host="romeo", node=1, gpus=1):
status = self.queue(host=host, user=user)
# VERBOSE(locals())
print(Printer.attribute(status, header=["Node", "Used GPUs"]))
#
# Required node not available (down, drained or reserved)
#
reserved = self.reserved_nodes(user=user)
def hostnames(host):
if host == "volta":
names = Parameter.expand("r-00[5-6]")
else:
names = Parameter.expand("r-00[3-4]")
max_gpus = 8 # this is for now hard coded
valid = []
for name in names:
if name not in reserved and status[name] + gpus <= max_gpus:
valid.append(name)
return valid
def find_random(host):
names = hostnames(host)
if len(names) == 0 or names is None:
return None
id = random.randint(0, len(host) - 1)
return names[id]
def find_first(host):
names = hostnames(host)
if names is None or len(names) == 0:
return None
else:
return names[0]
if node is None or node == "first":
node = find_first(host)
if node is None or node == "random":
node = find_random(host)
if node is not None:
Console.ok(f"Login on node {host}: {node}")
self.login(user=user, host=host, node=node, gpus=gpus)
else:
Console.error(f"not enough GPUs available: {host}: {node}")
def test_001_print(self):
HEADING()
pprint(self.data)
table = Printer.flatwrite(
self.data,
sort_keys=["name"],
order=["name", "address.street", "address.city"],
header=["Name", "Street", "City"],
)
print(table)
assert "Name" in str(table)
def delete(self, master=None, hosts=None):
# Delete master node
# TODO - k3s does not allow you to delete it's parent node
#if master is not None:
# banner(f"Deleting Master Node: {master}")
#
# command = Installer.oneline(f"""
# sudo kubectl delete {master}
# """)
# jobSet = JobSet("kubernetes_master_delete", executor=JobSet.ssh)
# jobSet.add({"name": self.hostname, "host": master, "command": command})
# jobSet.run()
# Uninstall workers
if hosts is not None:
workers = ', '.join(hosts)
banner(f"Deleting Worker Node(s): {workers}")
jobSet = JobSet("kubernetes_worker_delete", executor=JobSet.ssh)
for host in hosts:
command = Installer.oneline(f"""
sudo kubectl drain {host} --ignore-daemonsets --delete-local-data;
sudo kubectl delete node {host}
""")
jobSet.add({
"name": self.hostname,
"host": self.hostname,
"command": command
})
jobSet.run(parallel=len(hosts))
print(
"Workers:",
Printer.write(jobSet.array(),
order=[
"name", "command", "status", "stdout",
"returncode"
]))
t = JobSet("ssh", executor=JobSet.ssh)
t.add({"name": hostname, "host": hostname, "command": "pwd"})
t.run()
t.Print()
t = JobSet("onejob", executor=JobSet.ssh)
t.add({"name": hostname, "host": hostname, "command": "pwd", "os": True})
t.run()
t.Print()
t = JobSet("onejob", executor=JobSet.ssh)
t.add({"name": hostname, "host": hostname, "command": "pwd", "os": False})
t.run()
t.Print()
t = JobSet("onejob", executor=JobSet.ssh)
t.add({"name": hostname, "host": "red", "command": "pwd", "os": False})
t.run()
t.Print()
hostname = os.uname()[1]
t = JobSet("onejob", executor=JobSet.ssh)
for host in Parameter.expand("red,red[01-03]"):
t.add({"name": host, "host": host, "command": "uname -a"})
t.add({"name": hostname, "host": hostname, "command": "uname -a"})
t.run(parallel=3)
print(Printer.write(t.array(),
order=["name", "command", "status", "stdout",
"returncode"]))
details = USB.get_from_diskutil()
else:
details = USB.get_from_dmesg()
devices = yaml.safe_load(Printer.write(details,
order=[
"dev",
"info",
"formatted",
"size",
"active",
"readable",
"empty",
"direct-access",
"removable",
"writeable"],
header=[
"Path",
"Info",
"Formatted",
"Size",
"Plugged-in",
"Readable",
"Empty",
"Access",
"Removable",
"Writeable"],
output="yaml"))
layout = []
background = '#64778d'
layout.append([sg.Button('',
def benchmark(cls,
sysinfo=True,
csv=True,
prefix="# csv",
tag=None,
sum=True):
"""
prints out all timers in a convenient benchmark table
:return:
:rtype:
"""
#
# PRINT PLATFORM
#
print()
data_platform = systeminfo()
data_platform['cpu_count'] = multiprocessing.cpu_count()
if sysinfo:
print(Printer.attribute(
data_platform,
order=["Machine Attribute", "Value"],
output="table"
))
#
# PRINT TIMERS
#
timers = StopWatch.keys()
if len(timers) > 0:
data_timers = {}
for timer in timers:
data_timers[timer] = {
'start': time.strftime("%Y-%m-%d %H:%M:%S",
time.gmtime(
StopWatch.timer_start[timer])),
'time': StopWatch.get(timer, digits=3),
'sum': StopWatch.sum(timer, digits=3),
'status': StopWatch.get_status(timer),
'timer': timer,
'tag': tag or ''
}
for attribute in ["uname.node",
"user",
"uname.system",
"uname.machine",
"platform.version",
"sys.platform"]:
data_timers[timer][attribute] = data_platform[attribute]
# print(Printer.attribute(data_timers, header=["Command", "Time/s"]))
if 'benchmark_start_stop' in data_timers:
del data_timers['benchmark_start_stop']
for key in data_timers:
if key != 'benchmark_start_stop' and data_timers[key][
'status'] == None:
data_timers[key]['status'] = "failed"
elif data_timers[key]['status'] != None and data_timers[key][
'status'] == True:
data_timers[key]['status'] = "ok"
order = [
"timer",
"status",
"time",
"sum",
"start",
"tag",
"uname.node",
"user",
"uname.system",
"platform.version"
]
header = [
"Name",
"Status",
"Time",
"Sum",
"Start",
"tag",
"Node",
"User",
"OS",
"Version"
]
print()
print(Printer.write(
data_timers,
order=order,
header=header,
output="table"
))
print()
if csv:
if prefix is not None:
for entry in data_timers:
data_timers[entry]["# csv"] = prefix
order = ["# csv"] + order
print(Printer.write(
data_timers,
order=order,
header=header,
output="csv"
))
else:
pprint(data_timers)
print(Printer.write(
data_timers,
order=order[1:],
output="csv"
))
else:
print("ERROR: No timers found")
def do_burn(self, args, arguments):
"""
::
Usage:
burn gui [--hostname=HOSTNAME]
[--ip=IP]
[--ssid=SSID]
[--wifipassword=PSK]
[--bs=BLOCKSIZE]
[--dryrun]
[--no_diagram]
burn ubuntu NAMES [--inventory=INVENTORY] [--ssid=SSID] [-f]
[--wifipassword=PSK] [-v] --device=DEVICE [--country=COUNTRY]
[--upgrade]
burn raspberry NAMES --device=DEVICE
[--inventory=INVENTORY]
[--ssid=SSID]
[--wifipassword=PSK]
[--country=COUNTRY]
[--password=PASSWORD]
[-v]
[-f]
burn firmware check
burn firmware update
burn install
burn load --device=DEVICE
burn format --device=DEVICE
burn imager [TAG...]
burn mount [--device=DEVICE] [--os=OS]
burn unmount [--device=DEVICE] [--os=OS]
burn network list [--ip=IP] [--used]
burn network
burn info [--device=DEVICE]
burn image versions [--details] [--refresh] [--yaml]
burn image ls
burn image delete [--image=IMAGE]
burn image get [--url=URL] [TAG...]
burn backup [--device=DEVICE] [--to=DESTINATION]
burn copy [--device=DEVICE] [--from=DESTINATION]
burn shrink [--image=IMAGE]
burn cluster --device=DEVICE --hostname=HOSTNAME
[--burning=BURNING]
[--ip=IP]
[--ssid=SSID]
[--wifipassword=PSK]
[--bs=BLOCKSIZE]
[--os=OS]
[-y]
[--imaged]
[--set_passwd]
burn create [--image=IMAGE]
[--device=DEVICE]
[--burning=BURNING]
[--hostname=HOSTNAME]
[--ip=IP]
[--sshkey=KEY]
[--blocksize=BLOCKSIZE]
[--passwd=PASSWD]
[--ssid=SSID]
[--wifipassword=PSK]
[--format]
[--tag=TAG]
[--inventory=INVENTORY]
[--name=NAME]
[-y]
burn sdcard [TAG...] [--device=DEVICE] [-y]
burn set [--hostname=HOSTNAME]
[--ip=IP]
[--key=KEY]
[--keyboard=COUNTRY]
[--cmdline=CMDLINE]
burn enable ssh
burn wifi --ssid=SSID [--passwd=PASSWD] [--country=COUNTRY]
burn check [--device=DEVICE]
burn mac --hostname=HOSTNAME
Options:
-h --help Show this screen.
--version Show version.
--image=IMAGE The image filename,
e.g. 2019-09-26-raspbian-buster.img
--device=DEVICE The device, e.g. /dev/sdX
--hostname=HOSTNAME The hostnames of the cluster
--ip=IP The IP addresses of the cluster
--key=KEY The name of the SSH key file
--blocksize=BLOCKSIZE The blocksise to burn [default: 4M]
--burning=BURNING The hosts to be burned
Arguments:
TAG Keyword tags to identify an image
Files:
This is not fully thought through and needs to be documented
~/.cloudmesh/images
Location where the images will be stored for reuse
Description:
cms burn create --inventory=INVENTORY --device=DEVICE --name=NAME
Will refer to a specified cloudmesh inventory file (see cms help inventory).
Will search the configurations for NAME inside of INVENTORY and will burn
to DEVICE. Supports parameter expansion.
cms burn create --passwd=PASSWD
if the passwd flag is added the default password is
queried from the commandline and added to all SDCards
if the flag is omitted login via the password is
disabled and only login via the sshkey is allowed
Network
cms burn network list
Lists the ip addresses that are on the same network
+------------+---------------+----------+-----------+
| Name | IP | Status | Latency |
|------------+---------------+----------+-----------|
| Router | 192.168.1.1 | up | 0.0092s |
| iPhone | 192.168.1.4 | up | 0.061s |
| red01 | 192.168.1.46 | up | 0.0077s |
| laptop | 192.168.1.78 | up | 0.058s |
| unkown | 192.168.1.126 | up | 0.14s |
| red03 | 192.168.1.158 | up | 0.0037s |
| red02 | 192.168.1.199 | up | 0.0046s |
| red | 192.168.1.249 | up | 0.00021s |
+------------+----------------+----------+-----------+
cms burn network list [--used]
Lists the used ip addresses as a comma separated parameter
list
192.168.50.1,192.168.50.4,...
cms burn network address
Lists the own network address
+---------+----------------+----------------+
| Label | Local | Broadcast |
|---------+----------------+----------------|
| wlan0 | 192.168.1.12 | 192.168.1.255 |
+---------+----------------+----------------+
cms burn firmware check
Checks if the firmware on the Pi is up to date
cms burn firmware update
Checks and updates the firmware on the Pi
cms burn install
Installs a program to shrink img files. THis is
useful, after you created a backup to make the
backup smaller and allow faster burning in case of
recovery
This command is not supported on MacOS
cms burn load --device=DEVICE
Loads the sdcard into the USB drive. Thi sis similar to
loading a cdrom drive. It s the opposite to eject
cms burn format --device=DEVICE
Formats the SDCard in the specified device. Be
careful it is the correct device. cms burn info
will help you to identifying it
cms burn mount [--device=DEVICE] [--os=OS]
Mounts the file systems available on the SDCard
cms burn unmount [--device=DEVICE] [--os=OS]
Unmounts the mounted file systems from the SDCard
cms burn info [--device=DEVICE]
Provides useful information about the SDCard
cms burn image versions [--refresh] [--yaml]
The images that you like to burn onto your SDCard
can be cached locally with the image command. The
available images for the PI can be found when
using the --refresh option. If you do not specify
it it reads a copy of the image list from our
cache
cms burn image ls
Lists all downloaded images in our cache. You can
download them with the cms burn image get command
cms burn image delete [--image=IMAGE]
Deletes the specified image. The name can be found
with the image ls command
cms burn image get [--url=URL] [TAG...]
Downloads a specific image or the latest
image. The tag are a number of words separated by
a space that must occur in the tag that you find
in the versions command
cms burn backup [--device=DEVICE] [--to=DESTINATION]
This command requires you to install pishrink previously with
cms burn install
Backs up a SDCard to the given location.
cms burn copy [--device=DEVICE] [--from=DESTINATION]
Copies the file form the destination on the SDCard
this is the same as the SDCard command. we will in
future remove one
cms burn shrink [--image=IMAGE]
Shrinks the size of a backup or image file that
is on your local file system. It can only be used
for .img files
This command is not supported on MacOS.
cms burn create [--image=IMAGE]
[--device=DEVICE]
[--hostname=HOSTNAME]
[--ip=IP]
[--sshkey=KEY]
[--blocksize=BLOCKSIZE]
[--passwd=PASSWD]
[--ssid=SSID]
[--wifipassword=PSK]
[--format]
This command not only can format the SDCard, but
also initializes it with specific values
cms burn sdcard [TAG...] [--device=DEVICE]
this burns the sd card, see also copy and create
cms burn set [--hostname=HOSTNAME]
[--ip=IP]
[--key=KEY]
[--mount=MOUNTPOINT]
[--keyboard=COUNTRY]
[--cmdline=CMDLINE]
Sets specific values on the sdcard after it
has ben created with the create, copy or sdcard
command
a --ssh is missing from this command
cms burn enable ssh [--mount=MOUNTPOINT]
Enables the ssh server once it is booted
cms burn wifi --ssid=SSID [--passwd=PASSWD] [--country=COUNTRY]
Sets the wifi ssid and password after the card
is created, copied, or the sdcard is used.
The option country option expects an ISO 3166-1
two digit country code. The default is "US" and
the option not required if suitable. See
https://en.wikipedia.org/wiki/ISO_3166-1 for other
countries.
cms burn check [--device=DEVICE]
Lists the parameters that were set
with the set or create command
Examples: ( \\ is not shown)
> cms burn create --image=2019-09-26-raspbian-buster-lite
> --device=/dev/mmcblk0
> --hostname=red[5-7]
> --ip=192.168.1.[5-7]
> --sshkey=id_rsa
> cms burn image get latest
> cms burn image get https://downloads.raspberrypi.org/
> raspbian_lite/images/
> raspbian_lite-2018-10-11/2018-10-09-raspbian-stretch-lite.zip
> cms burn image delete 2019-09-26-raspbian-buster-lite
"""
map_parameters(arguments, "details", "refresh", "device", "dryrun",
"burning", "hostname", "ip", "sshkey", "blocksize",
"ssid", "url", "imaged", "key", "keyboard", "passwd",
"wifipassword", "version", "to", "os", "country",
"inventory", "name", "bs", "set_passwd", "cmdline",
"upgrade", "no_diagram")
# arguments.MOUNTPOINT = arguments["--mount"]
arguments.FORMAT = arguments["--format"]
arguments.FROM = arguments["--from"]
arguments.IMAGE = arguments["--image"]
arguments.output = "table" # hard code for now
arguments.bs = arguments.bs or "4M"
arguments.yes = arguments["-y"]
if len(arguments.TAG) == 0:
arguments.TAG = "latest"
# VERBOSE(arguments)
def execute(label, function):
StopWatch.start(label)
result = function
StopWatch.stop(label)
StopWatch.status(label, True)
return result
burner = Burner()
sdcard = SDCard()
if arguments.imager:
arguments.TAG = arguments.TAG or ["latest-lite"]
Console.msg(f"Tags: {arguments.TAG}")
try:
file = Imager.fetch(tag=arguments.TAG)
except: # noqa: E722
pass
try:
Imager.launch(file=file)
except Exception as e:
Console.error(
f"could not find image with the tag {arguments.TAG}\n\n{e}\n"
)
return ""
elif arguments.gui:
from cloudmesh.burn.gui import Gui
VERBOSE(arguments)
g = Gui(hostname=arguments.hostname,
ip=arguments.ip,
dryrun=arguments.dryrun,
no_diagram=arguments.no_diagram)
g.run()
return ""
elif arguments.raspberry:
banner(txt="RaspberryOS Burn", figlet=True)
if arguments.inventory:
inv_path = path_expand(f'~/.cloudmesh/{arguments.inventory}')
try:
burner = RaspberryBurner(
inventory=inv_path,
ssid=arguments['--ssid'],
wifipassword=arguments['--wifipassword'],
country=arguments['--country'])
except:
Console.error('Burner Error')
return ""
else:
try:
burner = RaspberryBurner(
names=arguments.NAMES,
ssid=arguments['--ssid'],
wifipassword=arguments['--wifipassword'],
force_inv=arguments['-f'],
country=arguments['--country'])
except Exception as e:
Console.error('Burner Error')
raise e
return ""
execute(
"burn raspberry",
burner.multi_burn(
names=arguments.NAMES,
devices=arguments.device,
verbose=arguments['-v'],
password=arguments['--password'],
))
return ""
elif arguments.ubuntu:
banner(txt="Ubuntu Burn with cloud-init", figlet=True)
names = Parameter.expand(arguments.NAMES)
if len(Parameter.expand(arguments.device)) > 1:
Console.error(
"Too many devices specified. Please only specify one")
return ""
if arguments.inventory:
c = Configure(inventory=arguments.inventory,
debug=arguments['-v'])
inv = Inventory(filename=arguments.inventory)
else:
names = Parameter.expand(arguments.NAMES)
manager, workers = Host.get_hostnames(names)
if workers:
worker_base_name = ''.join(
[i for i in workers[0] if not i.isdigit()])
cluster_name = manager or worker_base_name
inventory = path_expand(
f'~/.cloudmesh/inventory-{cluster_name}.yaml')
if not os.path.exists(inventory) or arguments['-f']:
if not manager:
Console.error("No inventory found. Can not create an "
"inventory without a "
"manager.")
return ""
Inventory.build_default_inventory(
filename=inventory,
manager=manager,
workers=workers,
manager_image='ubuntu-20.10-64-bit',
worker_image='ubuntu-20.10-64-bit')
c = Configure(inventory=inventory,
debug=arguments['-v'],
download_images=True)
inv = Inventory(filename=inventory)
names = Parameter.expand(arguments.NAMES)
manager, workers = Host.get_hostnames(names)
if manager:
if not arguments.ssid and 'wifi' in c.configs[manager][
'services']:
arguments.ssid = get_ssid()
if arguments.ssid == "":
Console.info('Could not determine SSID, skipping wifi '
'config')
arguments.ssid = None
if not arguments.wifipassword and arguments.ssid is not None:
arguments.country = Shell.locale().upper()
arguments.wifipassword = getpass(
f"Using --SSID="
f"{arguments.ssid} and "
f" --COUNTRY="
f"{arguments.country}, please "
f"enter wifi password:"******""
if 'ubuntu' not in tag:
Console.error(
"This command only supports burning ubuntu cards")
return ""
sdcard = SDCard(card_os="ubuntu")
# Code below taken from arguments.sdcard
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
# determine if we are burning a manager, as this needs to be done
# first to get the ssh public key
# manager = False
# for name in names:
# if not inv.has_host(name):
# Console.error(f'Could not find {name} in inventory {inv.filename}')
# return ""
# service = inv.get(name=name, attribute='service')
# if service == 'manager' and not manager:
# manager = name
# # make manager first in names
# names.remove(name)
# names.insert(0, name)
# elif service == 'manager' and manager:
# raise Exception('More than one manager detected in NAMES')
for name in names:
if not yn_choice(
f'Is the card to be burned for {name} inserted?'):
if not yn_choice(
f"Please insert the card to be burned for {name}. "
"Type 'y' when done or 'n' to terminante"):
Console.error("Terminating: User Break")
return ""
service = inv.get(name=name, attribute='service')
# Make sure bridge is only enabled if WiFi enabled
if service == 'manager':
services = inv.get(name=name, attribute='services')
if 'bridge' in services and not arguments.ssid:
Console.error(
'Service bridge can only be configured if WiFi'
' is enabled with --ssid and --wifipassword')
return ""
else:
enable_bridge = 'bridge' in services
Console.info(f'Burning {name}')
sdcard.format_device(device=arguments.device, yes=True)
sdcard.unmount(device=arguments.device)
sdcard.burn_sdcard(tag=tag, device=arguments.device, yes=True)
sdcard.mount(device=arguments.device, card_os="ubuntu")
if service == 'manager':
# Generate a private public key pair for the manager that will be persistently used
# priv_key, pub_key = c.generate_ssh_key(name)
# Write priv_key and pub_key to /boot/id_rsa and /boot/id_rsa.pub
# SDCard.writefile(filename=f'{sdcard.boot_volume}/id_rsa', content=priv_key)
# SDCard.writefile(filename=f'{sdcard.boot_volume}/id_rsa.pub', content=pub_key)
c.build_user_data(
name=name,
country=arguments.country,
upgrade=arguments.upgrade,
with_bridge=enable_bridge).write(
filename=sdcard.boot_volume + '/user-data')
c.build_network_data(name=name, ssid=arguments.ssid, password=arguments.wifipassword)\
.write(filename=sdcard.boot_volume + '/network-config')
else:
c.build_user_data(
name=name,
add_manager_key=manager,
upgrade=arguments.upgrade).write(
filename=sdcard.boot_volume + '/user-data')
c.build_network_data(name=name).write(
filename=sdcard.boot_volume + '/network-config')
time.sleep(
1
) # Sleep for 1 seconds to give ample time for writing to finish
sdcard.unmount(device=arguments.device, card_os="ubuntu")
Console.info("Remove card")
Console.ok(f"Burned {len(names)} card(s)")
return ""
elif arguments.firmware and arguments.check:
execute("firmware check", burner.firmware(action="check"))
return ""
elif arguments.firmware and arguments.update:
execute("firmware update", burner.firmware(action="update"))
return ""
if arguments.check:
execute("check", burner.check(device=arguments.device))
return ""
elif arguments.versions and arguments['image']:
StopWatch.start("image versions")
result = Image.create_version_cache(refresh=arguments["--refresh"])
output = "table"
if arguments["--yaml"]:
output = "yaml"
order = ["tag", 'date', "os", "type", 'version']
header = ["Tag", 'Date', "OS", "Type", 'Version']
if arguments.details:
order = ["tag", 'date', "os", "type", 'version', "url"]
header = ["Tag", 'Date', "OS", "Type", 'Version', "Url"]
print(
Printer.write(result,
order=order,
header=header,
output=output))
StopWatch.stop("image versions")
StopWatch.status("image versions", True)
return ""
elif arguments.load:
execute("load", sdcard.load_device(device=arguments.device))
return ""
elif arguments[
"format"]: # as format is a python word, we need to use an index
execute(
"format",
sdcard.format_device(device=arguments.device, unmount=True))
return ""
elif arguments.network and arguments["list"]:
if os_is_mac():
Console.error("Not yet implemented on MacOS")
return ""
ip = arguments.ip or Network.address()[0]['local']
details = Network.nmap(ip=ip)
if arguments.used:
print(','.join([x['ip'] for x in details]))
else:
print(
Printer.write(details,
order=[
'name',
"ip",
"status",
"latency",
],
header=[
'Name',
"IP",
"Status",
"Latency",
]))
return ""
elif arguments.network:
if os_is_mac():
Console.error("Not yet implemented on MacOS")
return ""
# print (Network.nmap())
details = Network.address()
print(
Printer.write(details,
order=['label', "local", "broadcast"],
header=["Label", "Local", "Broadcast"]))
return ""
elif arguments.wifi:
password = arguments.passwd
ssid = arguments.ssid or get_ssid()
country = arguments.country
if password is None:
password = getpass("Please enter the Wifi password or enter "
"for no password: "******"macos"
elif os_is_linux():
host = "linux"
elif os_is_pi():
host = "raspberry"
else:
Console.error(
"This command is not yet implemented for your OS")
return ""
burner.configure_wifi(ssid,
psk=password,
country=country,
host=host)
return ""
elif arguments.info:
output = arguments.output or "table"
card = SDCard()
execute("info", card.info(output=output))
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
return ""
elif arguments.install:
if os_is_mac():
Console.error("Not yet implemented on MacOS")
return ""
execute("install", burner.install())
return ""
elif arguments.shrink:
if os_is_mac():
Console.error("Not yet implemented on MacOS")
return ""
execute("shrink", burner.shrink(image=arguments.IMAGE))
return ""
elif arguments.backup:
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
execute(
"backup",
sdcard.backup(device=arguments.device, to_file=arguments.to))
return ""
elif arguments[
"copy"]: # as copy is a reserved word we need to use the index
USB.check_for_readers()
execute(
"copy",
sdcard.copy(device=arguments.device, from_file=arguments.FROM))
return ""
elif arguments.sdcard:
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
if arguments.device is None:
card = SDCard()
card.info()
Console.error("Please specify a device")
return ""
arguments.TAG = arguments.TAG or ["latest-lite"]
if any("ubuntu" in tag for tag in arguments.TAG):
sdcard = SDCard(card_os="ubuntu")
execute(
"format",
sdcard.format_device(device=arguments.device, unmount=True))
execute("unmount", sdcard.unmount(device=arguments.device))
execute(
"sdcard",
sdcard.burn_sdcard(tag=arguments.TAG,
device=arguments.device,
yes=arguments.yes))
return ""
elif arguments.mount:
if arguments.device is None:
card = SDCard
card.info()
Console.error("Please specify a device")
return ""
execute(
"mount",
sdcard.mount(device=arguments.device, card_os=arguments.os))
return ""
elif arguments.unmount:
card = SDCard(card_os=arguments.os)
execute(
"unmount",
card.unmount(device=arguments.device, card_os=arguments.os))
return ""
elif arguments.mac:
hostnames = Parameter.expand(arguments.hostname)
execute("mac", burner.mac(hostnames=hostnames))
return ""
elif arguments.set:
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
if arguments.hostname:
execute("set hostname",
burner.set_hostname(arguments.hostname))
if arguments.ip:
execute("set ip", burner.set_static_ip(arguments.ip))
if arguments.key:
execute("set key", burner.set_key(arguments.key))
if arguments.keyboard:
execute("set keyboard",
burner.keyboard(country=arguments.keyboard))
if arguments.cmdline:
execute("set cmdline", burner.set_cmdline(arguments.cmdline))
return ""
elif arguments.enable and arguments.ssh:
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
execute("enable ssh", burner.enable_ssh())
return ""
# elif arguments.versions and arguments.image:
# image = Image()
elif arguments.ls and arguments['image']:
execute("image ls", Image().ls())
return ""
elif arguments.delete and arguments.IMAGE:
execute("image rm", Image().rm(arguments.IMAGE))
return ""
elif arguments["get"] and arguments['image'] and arguments["--url"]:
image = Image()
execute("image fetch", image.fetch(url=arguments.url))
return ""
elif arguments["get"] and arguments['image'] and arguments["TAG"]:
tag = arguments["TAG"]
if "latest" in tag and ("full" in tag or "lite" in tag):
result = Image.create_version_cache(
refresh=arguments["--refresh"])
image = Image()
execute("image fetch", image.fetch(tag=arguments["TAG"]))
return ""
elif arguments["get"] and arguments['image']:
image = Image()
execute("image fetch", image.fetch(tag="latest"))
return ""
elif arguments.cluster:
# is true when
#
# cms burn cluster --hostname=red,red00[1-2]
# --device=/dev/sdb
# --ip=10.1.1.[1-3]
# --ssid=myssid
# --wifipassword=mypass
#
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
execute("cluster", burner.cluster(arguments=arguments))
return ""
elif arguments.create and arguments.inventory:
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
if not os_is_pi():
print()
Console.error(
"This command has only been written for a Raspberry Pis. "
"Terminating for caution")
print()
if yn_choice("Continue anyways?"):
pass
else:
return
if not arguments.name:
Console.error(
"Missing --name parameter. See cms help burn for usage")
return ""
if not arguments.device:
Console.error(
"Missing --device parameter. See cms help burn for usage")
return ""
StopWatch.start("burn inventory")
multi_burner = MultiBurner()
# Perhaps we want to change the path at some point
inventory = f"~/.cloudmesh/{arguments.inventory}"
multi_burner.burn_inventory(inventory=inventory,
name=arguments.name,
device=arguments.device,
yes=arguments.yes,
passwd=arguments.passwd)
StopWatch.stop("burn inventory")
StopWatch.status("burn inventory", True)
StopWatch.benchmark(sysinfo=False, csv=False)
return ""
elif arguments.create:
try:
USB.check_for_readers()
except Exception as e:
print()
Console.error(e)
print()
return ""
if arguments["--passwd"]:
passwd = arguments["--passwd"]
elif "PASSWD" in os.environ:
passwd = os.environ["PASSWD"]
else:
passwd = generate_strong_pass()
psk = None
if arguments["--ssid"]:
ssid = arguments["--ssid"]
if arguments["--wifipassword"]:
psk = arguments["--wifipassword"]
else:
psk = None
else:
if arguments["--wifipassword"]:
print("Can't have wifi password with no ssid")
return
else:
ssid = None
image = 'latest' or arguments.IMAGE
dev = os.environ['DEV'] if 'DEV' in os.environ else None
devices = arguments["--device"] or dev or None
if devices is not None:
devices = Parameter.expand_string(devices)
hostnames = Parameter.expand(arguments.hostname)
if arguments.burnimg is None:
burning = hostnames
else:
burning = arguments.burning
VERBOSE(arguments)
ips = None if not arguments.ip else Parameter.expand(arguments.ip)
key = arguments.sshkey
tag = arguments['--tag']
if os_is_pi() or os_is_linux():
blocksize = arguments.blocksize
StopWatch.start("total")
multi = MultiBurner()
multi.burn_all(
burning=burning,
image=image,
device=devices,
blocksize=blocksize,
progress=True,
hostnames=hostnames,
# not difference between names and name, maybe we should align
ips=ips,
key=key,
password=passwd,
ssid=ssid,
psk=psk,
tag=tag,
yes=arguments.yes)
StopWatch.stop("total")
StopWatch.status("total", True)
StopWatch.benchmark(sysinfo=False, csv=False)
else:
Console.error(
"This command is only supported ona Pi and Linux")
return ""
Console.error("see manual page: cms help burn")
return ""
if __name__ == '__main__':
#
# Tow ways to run.
#
# Static method
result = JobScript.execute("""
# This is a comment
pwd # tag: pwd
uname -a
""")
print(
Printer.write(
result,
order=["name", "command", "status", "stdout", "returncode"]))
# Variables
job = JobScript()
job.run(name="variable script",
script="""
# This is a comment
pwd # tag: pwd
uname -a
""")
print(
Printer.write(
job.array(),
order=["name", "command", "status", "stdout", "returncode"]))
def fetch(self, url=None, tag=None, verify=True):
"""
Download the image from the URL in self.image_name
If it is 'latest', download the latest image - afterwards use
cm-pi-burn image ls
to get the name of the downloaded latest image.
"""
if url is None:
data = Image().create_version_cache(refresh=False) # noqa: F841
image = Image().find(tag=tag)
if image is None:
Console.error("No matching image found.")
return None
elif len(image) > 1:
Console.error("Too many images found")
print(
Printer.write(image,
order=["tag", "version"],
header=["Tag", "Version"]))
return None
image = image[0]
# image_path = Image().directory + "/" + Image.get_name(image[
# "url"]) + ".img"
if not os.path.exists(self.directory):
os.makedirs(self.directory)
os.chdir(self.directory)
# get image URL metadata, including the name of the latest image after
# the 'latest' URL redirects to the URL of the actual image
if "ubuntu" in image["url"]:
source_url = requests.head(image["url"], allow_redirects=True).url
size = requests.get(image["url"], verify=False,
stream=True).headers['Content-length']
xz_filename = os.path.basename(source_url)
img_filename = xz_filename.replace('.xz', '')
img_file = Path(Path(self.directory) / Path(img_filename))
# xz_file = Path(Path(self.directory) / Path(xz_filename))
print(f"Downloading {xz_filename}")
os.chdir(path_expand("~/.cloudmesh/cmburn/images"))
if os.path.exists(img_filename):
print()
Console.warning(
f"The file is already downloaded. Found at:\n\n"
f" {img_file}\n")
return img_file
Shell.download(image["url"], xz_filename, provider='system')
# wget.download(image["url"], out=xz_filename)
# os.system(f'wget -O {xz_filename} {image["url"]}')
print(f"Extracting {img_filename}")
self.unzip_image(xz_filename)
try:
Path(xz_filename).unlink()
except: # noqa: E722
pass
return img_filename
else:
source_url = requests.head(image["url"], allow_redirects=True).url
size = requests.get(image["url"], verify=False,
stream=True).headers['Content-length']
zip_filename = os.path.basename(source_url)
img_filename = zip_filename.replace('.zip', '.img')
sha1_filename = zip_filename + '.sha1'
sha256_filename = zip_filename + '.sha256'
print(f"Downloading {zip_filename}")
img_file = Path(Path(self.directory) / Path(img_filename))
zip_file = Path(Path(self.directory) / Path(zip_filename))
# sha1_file = Path(Path(self.directory) / Path(sha1_filename))
# sha256_file = Path(Path(self.directory) / Path(sha256_filename))
# cancel if image already downloaded
if os.path.exists(img_filename):
print()
Console.warning(
f"The file is already downloaded. Found at:\n\n"
f" {img_file}\n")
return img_file
# download the image, unzip it, and delete the zip file
image['sha1'] = image['url'] + ".sha1"
image['sha256'] = image['url'] + ".sha256"
if verify:
Shell.download(image["sha1"], sha1_filename, provider='system')
Shell.download(image["sha256"],
sha256_filename,
provider='system')
# wget.download(image["sha1"], out=sha1_filename)
# wget.download(image["sha256"], out=sha256_filename)
# os.system(f'wget -O {sha1_filename} {image["sha1"]}')
# os.system(f'wget -O {sha256_filename} {image["sha256"]}')
Shell.download(image["url"], zip_filename, provider='system')
# wget.download(image["url"], out=zip_filename)
# os.system(f'wget -O {zip_filename} {image["url"]}')
if verify:
sha1 = sha1sum(zip_file)
sha256 = sha256sum(zip_filename)
f_sha1 = readfile(sha1_filename).split(" ")[0]
f_sha256 = readfile(sha256_filename).split(" ")[0]
if f_sha1 == sha1:
Console.ok("SHA1 is ok")
if f_sha256 == sha256:
Console.ok("SHA256 is ok")
zip_size = os.path.getsize(zip_file)
if int(size) != zip_size:
Console.error(
f"Repository reported zip size {size} does not equal "
f"download size {zip_size}. Please try again.")
return None
# if latest: # rename filename from 'latest' to the actual image name
# Path('raspbian_lite_latest').rename(zip_filename)
print(f"Extracting {img_filename}")
self.unzip_image(zip_filename)
Path(zip_filename).unlink()
return img_filename
def do_provider(self, args, arguments):
"""
::
Usage:
provider list [--output=OUTPUT]
provider info SERVICE NAME WHAT
Arguments:
NAME The name of the key.
Options:
--output=OUTPUT the format of the output [default: table]
Description:
What: output, sample
Examples:
Getting the sample and output from provides via a command
cms provider info compute openstack sample
cms provider info compute openstack output
cms provider list --output=json
cms provider list
q """
map_parameters(arguments, 'output')
if arguments.info:
try:
service = arguments.SERVICE
name = arguments.NAME
what = arguments.WHAT
services = find()
for provider in services:
try:
if provider['service'] == service and provider[
'name'] == name:
if arguments.WHAT == 'sample':
print(
textwrap.dedent(
provider["provider"].sample))
elif arguments.WHAT == 'output':
print(
json.dumps(provider["provider"].output,
indent=4))
print()
except Exception as e:
print(e)
except:
Console.error("Problem getting the Provider info")
return ""
elif arguments.list:
print(arguments.output)
_paths = find()
for entry in _paths:
del entry["provider"] # can not be printed
print(
Printer.write(_paths,
order=["service", "name", "active", "path"],
output=arguments.output))
return ""
def config(self, config):
Console.info("config")
print(Printer.attribute(config))
def burn_sdcard(self,
image=None,
tag=None,
device=None,
blocksize="4M",
name="the inserted card",
yes=False):
"""
Burns the SD Card with an image
:param image: Image object to use for burning (used by copy)
:type image: str
:param name:
:type name: str
:param tag: tag object used for burning (used by sdcard)
:type tag: str
:param device: Device to burn to, e.g. /dev/sda
:type device: str
:param blocksize: the blocksize used when writing, default 4M
:type blocksize: str
:param yes:
:type yes: str
"""
if image and tag:
Console.error("Implementation error, burn_sdcard can't have image "
"and tag.")
return ""
Console.info(f"Burning {name} ...")
if image is not None:
image_path = image
else:
image = Image().find(tag=tag)
if image is None:
Console.error("No matching image found.")
return ""
elif len(image) > 1:
Console.error("Too many images found")
print(
Printer.write(image,
order=["tag", "version"],
header=["Tag", "Version"]))
return ""
image = image[0]
if "ubuntu" in image["url"]:
_name = os.path.basename(Image.get_name(image["url"]))
_name = _name.replace(".xz", "")
else:
_name = os.path.basename(Image.get_name(image["url"])) + ".img"
image_path = Image().directory + "/" + _name
print(image_path)
if not os.path.isfile(image_path):
tags = ' '.join(tag)
print()
Console.error(
f"Image with tags '{tags}' not found. To download use")
print()
Console.msg(f"cms burn image get {tags}")
print()
return ""
orig_size = size = humanize.naturalsize(os.path.getsize(image_path))
# size = details[0]['size']
n, unit = size.split(" ")
unit = unit.replace("GB", "G")
unit = unit.replace("MB", "M")
n = float(n)
if unit == "G":
n = n * 1000**3
elif unit == "M":
n = n * 1000**2
size = int(n)
banner(f"Preparing the SDCard {name}")
print(f"Name: {name}")
print(f"Image: {image_path}")
print(f"Image Size: {orig_size}")
print(f"Device: {device}")
print(f"Blocksize: {blocksize}")
if os_is_mac():
blocksize = blocksize.lower()
print()
Sudo.password()
if device is None:
Console.error("Please specify a device")
return
#
# speedup burn for MacOS
#
if device.startswith("/dev/disk"):
device = device.replace("/dev/disk", "/dev/rdisk")
if os_is_mac():
details = USB.get_from_diskutil()
USB.print_details(details)
if not (yes or yn_choice(f"\nDo you like to write {name} on {device} "
f"with the image {image_path}")):
return ""
# TODO Gregor verify this is ok commenting out this line
# self.mount(device=device)
if os_is_mac():
command = f"sudo dd if={image_path} bs={blocksize} |" \
f' tqdm --bytes --total {size} --ncols 80 |' \
f" sudo dd of={device} bs={blocksize}"
else:
# command = f"sudo dd if={image_path} of={device} bs={blocksize} status=progress conv=fsync"
command = f"sudo dd if={image_path} bs={blocksize} oflag=direct |" \
f' tqdm --bytes --total {size} --ncols 80 |' \
f" sudo dd of={device} bs={blocksize} iflag=fullblock " \
f"oflag=direct conv=fsync"
print(command)
os.system(command)
Sudo.execute("sync")
if os_is_linux():
self.unmount(device=device, full=True)
else:
self.unmount(device=device)
def info(self,
print_os=True,
print_fdisk=True,
print_stdout=True,
output="table"):
"""
Finds out information about USB devices
TODO: should we rename print_stdout to debug? seems more in
line with cloudmesh
:param print_os:
:type print_os:
:param print_fdisk:
:type print_fdisk:
:param print_stdout: if set to True prints debug information
:type print_stdout: bool
:param output:
:type output:
:return: dict with details about the devices
:rtype: dict
"""
if print_os and print_stdout:
if os_is_pi():
banner("This is Raspberry PI")
elif os_is_mac():
banner("This is Mac")
elif os_is_windows():
banner("This is a Windows Computer")
elif os_is_linux():
banner("This is a Linux Computer")
else:
Console.error("unkown OS")
sys.exit(1)
if os_is_pi() and print_fdisk and print_stdout:
result = USB.fdisk("/dev/mmcblk0")
if print_stdout:
banner("Operating System SD Card")
print(result)
details = USB.get_from_usb()
if print_stdout and details is not None:
banner("USB Device Probe")
print(
Printer.write(details,
order=[
"address", "bus", "idVendor", "idProduct",
"hVendor", "hProduct", "iManufacturer",
"iSerialNumber", "usbVersion", "comment"
],
header=[
"Adr.", "bus", "Vendor", "Prod.", "H Vendor",
"H Prod.", "Man.", "Ser.Num.", "USB Ver.",
"Comment"
],
output=output))
# devices = USB.get_devices()
# banner("Devices found")
# print ('\n'.join(sorted(devices)))
if os_is_mac():
names = USB.get_dev_from_diskutil()
details = USB.get_from_diskutil()
else:
details = USB.get_from_dmesg()
if print_stdout:
banner("SD Cards Found")
if os_is_mac():
print("We found the follwing cards:")
print(" - /dev/" + "\n - /dev/".join(names))
print()
print("We found the follong file systems on these disks:")
print()
print(
Printer.write(details,
order=[
"dev", "info", "formatted", "size", "active",
"readable", "empty", "direct-access",
"removable", "writeable"
],
header=[
"Path", "Info", "Formatted", "Size",
"Plugged-in", "Readable", "Empty", "Access",
"Removable", "Writeable"
],
output=output))
# lsusb = USB.get_from_lsusb()
# from pprint import pprint
# pprint (lsusb)
# endors = USB.get_vendor()
# print(vendors)
# udev = subprocess.getoutput("udevadm info -a -p $(udevadm info -q path -n /dev/sda)")
#
# attributes = ["vendor","model", "model", "version", "manufacturer",
# "idProduct", "idVendor"]
# for line in udev.splitlines():
# if any(word in line for word in attributes):
# print(line)
if print_stdout:
if os_is_linux():
card = SDCard()
m = card.ls()
banner("Mount points")
if len(m) != 0:
print(
Printer.write(m,
order=[
"name", "path", "type", "device",
"parameters"
],
header=[
"Name", "Path", "Type", "Device",
"Parameters"
],
output=output))
else:
Console.warning(
"No mount points found. Use cms burn mount")
print()
# Convert details into a dict where the key for each entry is the device
details = {detail['dev']: detail for detail in details}
return details
def create(self, **kwargs):
arguments = dotdict(kwargs)
name = arguments.name
cloud = arguments.cloud
if name is None:
name_generator = Name()
vms = [str(name_generator)]
else:
vms = self.expand(name)
#
# Step 0, find the cloud
#
variables = Variables()
if cloud is None:
arguments.cloud = cloud = variables['cloud']
# Step 1. iterate through the names to see if they already exist in
# the DB and fail if one of them already exists
database = CmDatabase()
defaults = Config()[f"cloudmesh.cloud.{cloud}.default"]
duplicates = []
for vm in vms:
query = {"name": vm}
duplicates += database.find(collection=f'{cloud}-node',
query=query)
database.close_client()
if len(duplicates) > 0:
print(
Printer.flatwrite(duplicates,
order=['cm.name', 'cm.cloud'],
header=['Name', 'Cloud'],
output='table'))
raise Exception("these vms already exists")
# Step 2. identify the image and flavor from kwargs and if they do
# not exist read them for that cloud from the yaml file
if arguments.image is None:
arguments.image = self.find_attribute('image',
[variables, defaults])
if arguments.image is None:
raise ValueError("image not specified")
if arguments.group is None:
arguments.group = self.find_attribute('group',
[variables, defaults])
if arguments.group is None:
arguments.group = "default"
if arguments.size is None:
arguments.size = self.find_attribute('size', [variables, defaults])
if arguments.size is None:
raise ValueError("size not specified")
# Step 3: use the create command to create the vms
# created = self.loop(vms, self.p.create, **arguments)
arguments['name'] = vms
created = self.loop(self._create, **arguments)
VERBOSE(created)
self.list()
return created