def load_data(self): self.background = '#64778d' if os_is_mac(): self.details = USB.get_from_diskutil() else: self.details = USB.get_from_dmesg() self.devices = yaml.safe_load(Printer.write(self.details, order=[ "dev", "info", "formatted", "size", "active", "readable", "empty", "direct-access", "removable", "writeable"], header=[ "Path", "Info", "Formatted", "Size", "Plugged-in", "Readable", "Empty", "Access", "Removable", "Writeable"], output="yaml"))
def size(device="/dev/sdX"): size = 64 * 1000**3 # this is a bug as we need that for Linux and PI if os_is_mac(): result = Shell.run("diskutil list external").splitlines() for line in result: if "FDisk_partition_scheme" in line: data = line.split() size, unit = data[2].replace("*", ""), data[3] if unit == "GB": size = int(float(size) * 1000**3) else: size = 64 Console.error("Unit not GB") break elif os_is_linux(): try: result = Shell.run( f"sudo blockdev --getsize64 {device}").strip() result = int(result) except Exception as e: # noqa: F841 Console.error( f"Could not determine size of the device {device}") sys.exit() return size
def probe_os(self): if os_is_mac(): where_raspberry = location(host_os="macos", card_os="raspberry", volume="boot") where_ubuntu = location(host_os="macos", card_os="ubuntu", volume="boot") elif os_is_pi(): where_raspberry = location(host_os="raspberry", card_os="raspberry", volume="boot") where_ubuntu = location(host_os="raspberry", card_os="ubuntu", volume="boot") elif os_is_linux(): where_raspberry = location(host_os="ubuntu", card_os="raspberry", volume="boot") where_ubuntu = location(host_os="ubuntu", card_os="ubuntu", volume="boot") else: return "unkwn" if os.path.exists(where_raspberry): return "raspberryos" elif os.path.exists(where_ubuntu): return "ubuntu" else: return "unkown"
def writefile(filename=None, content=None, append=False): """ Writes the content in the the given file. :param filename: the filename :type filename: str :param content: the content :type content: str :param append: if true it append it at the end, otherwise the file will be overwritten :type append: bool :return: the output created by the write process :rtype: int """ os.system("sync") if append: content = Sudo.readfile(filename, split=False, decode=True) + content command = f"echo '{content}' | sudo cp /dev/stdin {filename}" if os_is_mac() and "\0" in command: command = command.replace("\0", "") os.system(command) os.system("sync") return content
def launch(file=None): if file is not None: if not str(file).endswith(".img"): raise ValueError(f"file {file} does not end with .img") if not os.path.exists(file): raise ValueError(f"image file {file} does not exist") elif file is None: file = "" Imager.install() if os_is_linux() or os_is_pi(): Sudo.password() os.system(f"sudo rpi-imager {file}") elif os_is_mac(): os.system( f"/Applications/Raspberry\ Pi\ Imager.app/Contents/MacOS/rpi-imager {file} " # noqa: W605 "> /dev/null 2>&1") # noqa: W605 elif os_is_windows(): os.chdir(r"C:\Program Files (x86)\Raspberry Pi Imager") os.system("rpi-imager.exe")
def install(force=False): if os_is_mac(): return if not Imager.installed() or force: if os_is_linux() or os_is_pi(): Sudo.password() os.system("sudo apt uninstall -y rpi-imager") else: Console.warning("Installation is not supported")
def check_for_readers(): if os_is_mac(): readers = USB.get_dev_from_diskutil() if len(readers) == 0: print() Console.error("Please make sure the reader and the SDCard " "are properly inserted") raise ValueError("No card found") elif len(readers) > 1: print()
def get_devices(): """ finds all devices starting with /dev/sd? :return: list :rtype: list of devices """ if os_is_mac(): return glob.glob("/dev/disk?") else: return glob.glob("/dev/sd?")
def unmount(self, device=None, card_os="raspberry", full=False): """ Unmounts the current SD card. param full indicates whether to use -t flag :param device: device to unmount, e.g. /dev/sda :type device: str :param card_os: :type card_os: :param full: :type full: :return: :rtype: """ # self.card_os = card_os os.system('sudo sync') # flush any pending/in-process writes os.system("sync") if os_is_linux() or os_is_pi(): Sudo.password() if full: _execute(f"eject {device}", f"sudo eject {device}") else: # _execute(f"eject {device}", f"sudo eject -t {device}") device_basename = os.path.basename(device) result = Shell.run('lsblk') if device_basename in result.split(): for line in result.splitlines(): line = line.split() if device_basename in line[0] and len(line) > 6: Console.ok(f'sudo umount {line[6]}') os.system(f'sudo umount {line[6]}') # _execute(f"unmounting {self.boot_volume}", f"sudo umount {self.boot_volume}") # _execute(f"unmounting {self.root_volume}", f"sudo umount {self.root_volume}") elif os_is_mac(): _execute(f"unmounting {self.boot_volume}", f"diskutil umountDisk {device}") else: Console.error("Not yet implemented for your OS") return "" os.system("sync") # rm = [f"sudo rmdir {self.boot_volume}", # f"sudo rmdir {self.root_volume}"] # for command in rm: # _execute(command, command) return True
def fdisk(dev): """ calls fdisk on the specified device :param dev: device, example /dev/sdz :type dev: str :return: the output from the fdisk command :rtype: str """ if os_is_mac(): raise NotImplementedError("fdisk -l not supported on MacOS") else: Sudo.password() return subprocess.getoutput(f"sudo fdisk -l {dev}")
def install(self): """ Installs /usr/local/bin/pishrink.sh Installs parted :return: :rtype: """ if os_is_mac(): Console.error("This command is not supported on MacOS") return "" else: banner("Installing pishrink.sh into /usr/local/bin") script_name = Shell.download( 'https://raw.githubusercontent.com/Drewsif/PiShrink/master/pishrink.sh', 'pishrink.sh', provider='system') script = \ f""" chmod +x {script_name} sudo mv {script_name} /usr/local/bin """ result = JobScript.execute(script) print( Printer.write(result, order=[ "name", "command", "status", "stdout", "returncode" ])) if os_is_linux() or os_is_pi(): banner("Installing pishrink.sh into /usr/local/bin") script = \ """ sudo apt install parted -y > $HOME/tmp.log """ result = JobScript.execute(script) print( Printer.write(result, order=[ "name", "command", "status", "stdout", "returncode" ]))
def get_dev_from_diskutil(): if os_is_mac(): import plistlib external = subprocess.check_output("diskutil list -plist external".split(" ")) r = dict(plistlib.loads(external)) details = [] if len(r['AllDisksAndPartitions']) == 0: Console.error("No partition found") return "" else: for dev in r['AllDisksAndPartitions']: details.append(dev['DeviceIdentifier']) return details else: return None
def _readfile(filename=None, split=False, trim=False, decode=True): """ Reads the content of the file as sudo and returns the result :param filename: the filename :type filename: str :param split: if true returns a list of lines :type split: bool :param trim: trim trailing whitespace. This is useful to prevent empty string entries when splitting by '\n' :type trim: bool :param decode: :type decode: bool :return: the content :rtype: str or list """ Sudo.execute("sync") if os_is_mac(): if decode: mode = "r" else: mode = "rb" content = common_readfile(filename, mode=mode) else: Sudo.password() result = Sudo.execute(f"cat {filename}", decode=decode) content = result.stdout if trim: content = content.rstrip() if split: content = content.splitlines() Sudo.execute("sync") return content
def detect(): if os_is_mac(): details = USB.get_from_diskutil() else: details = USB.get_from_dmesg() return details
def create_layout(self): def line(msg): title = f'__ {msg} ' title = title.ljust(120, '_') burn_layout.append([sg.Text(title)]) location = os.path.dirname(os.path.abspath(inspect.getsourcefile(Gui))) cm_logo = f'{location}/images/cm-logo-100.png' pi_logo = f'{location}/images/raspberry-logo-white-100.png' rack_file = f"~/.cloudmesh/gui/{self.manager}-rack.png" net_file = f"~/.cloudmesh/gui/{self.manager}-net.png" burn_layout = [ [sg.T('')] ] if not self.no_diagram: net_layout = [ [sg.Image(data=image(net_file), key='net-image', background_color='white')] ] rack_layout = [ [sg.Image(data=image(rack_file), key='rack-image', background_color='white')] ] else: net_layout = [] rack_layout = [] size = log_size log_layout = [[sg.T('', key="log", size=size)]] log_scroll_layout = [ [sg.Column(layout=log_layout, scrollable=True)] ] burn_layout.append( [ sg.Text(40 * " "), sg.Image(data=image(cm_logo), key='cm-logo'), sg.Image(data=image(pi_logo), key='pi-logo') ] ) # # DEVICES # line("Devices") if os_is_mac(): devices = USB.get_dev_from_diskutil() elif os_is_linux(): devices = USB.get_from_dmesg() else: devices = {} count = 0 for device in devices: default = count == 0 if os_is_linux(): burn_layout.append( [sg.Radio(device['dev'], group_id="DEVICE", default=default, key=f"device-{device['name']}")] ) else: burn_layout.append( [sg.Radio(device, group_id="DEVICE", default=default, key=f"device-{device}")] ) count = count + 1 # # SDCARD # line("SD Card") burn_layout.append( [sg.Checkbox("format", default=True, key="imaged")] ) # # OPERATING SYSTEM # line("Operating System") count = 0 for entry in image_tags: data = image_tags[entry] name = data["name"] default = count == 0 burn_layout.append( [sg.Radio(name, group_id="OS", enable_events=True, default=default, key=entry)] ) count = count + 1 # # SECURITY # line("Security") if self.key is not None: burn_layout.append([sg.Text("Key", size=security_width), sg.Input(key="key", default_text=self.key)]) burn_layout.append([sg.Text("SSID", size=security_width), sg.Input(key="ssid", default_text=self.ssid)]) burn_layout.append( [sg.Text("Wifi Password", size=security_width), sg.Input(key="wifi", default_text="", password_char='*')]) # # MANAGER # line("Manager") if self.manager is not None: manager = self.manager i = 0 burn_layout.append( [ sg.Text('', size=status_width, key=str(f'status-{manager}')), sg.Button('Burn', key=str(f'button-{manager}')), sg.Text(manager, size=name_width), sg.Text("manager", size=name_width), sg.Input(default_text=manager, size=name_width, key=str(f'name-{manager}')), sg.Input(default_text=self.ips[i], size=name_width, key=str(f'ip-{manager}')), sg.Text('Image'), sg.Input(default_text="latest-full", size=tag_width, key=str(f'tags-{manager}')) ] ) # # WORKERS # line("Workers") worker_layout = [] if self.workers is not None: i = 1 for worker in self.workers: worker_layout.append( [ sg.Text('', size=status_width, key=str(f'status-{worker}')), sg.Button('Burn', key=str(f'button-{worker}')), sg.Text(worker, size=name_width), sg.Text("worker", size=name_width), sg.Input(default_text=worker, size=name_width, key=str(f'name-{worker}')), sg.Input(default_text=self.ips[i], size=name_width, key=str(f'ip-{worker}')), sg.Text('Image'), sg.Input(default_text="latest-lite", size=tag_width, key=str(f'tags-{worker}')), ]) i = i + 1 burn_layout.append([sg.Column(worker_layout, scrollable=True, size=(800, 400))]) # # TABS # self.cancel_layout = [sg.Button('Cancel', key="cancel")] self.layout = [ self.cancel_layout, # , sg.Button(' Next Card -> ', key="next")], [ sg.TabGroup( [ [ sg.Tab('Burn', burn_layout, key="panel-burn"), sg.Tab('Log', log_scroll_layout, key="panel-lg", background_color='white'), sg.Tab('Network', net_layout, key="panel-net", background_color='white'), sg.Tab('Rack', rack_layout, key="panel-rack", background_color='white') ] ], tooltip='Rack', key="mytabs") ] ] return self.layout
def format_device(self, device='dev/sdX', unmount=True, yes=False, verbose=True): """ Formats device with one FAT32 partition WARNING: make sure you have the right device, this command could potentially erase your OS :param device: The device on which we format :type device: str :param unmount: :type unmount: :param yes: :type yes: :param verbose: :type verbose: :return: :rtype: """ _title = "UNTITLED" def prepare_sdcard(): """ ensures a card is detected and unmounted :return: True if prepared :rtype: bool """ # Console.ok(f'sudo eject -t {device}') os.system(f'sudo eject -t {device}') time.sleep(3) device_basename = os.path.basename(device) result = Shell.run('lsblk') if device_basename in result.split(): for line in result.splitlines(): line = line.split() if device_basename in line[0] and len(line) > 6: Console.ok(f'sudo umount {line[6]}') os.system(f'sudo umount {line[6]}') return True else: Console.error("SD Card not detected. Please reinsert " "card reader. ") if not yn_choice("Card reader re-inserted? No to cancel " "operation"): return False else: time.sleep(3) return prepare_sdcard() Sudo.password() if os_is_linux() or os_is_pi(): if verbose: banner(f"format {device}") else: print(f"format {device}") if not prepare_sdcard(): return False # TODO Gregor verify commenting out the below is ok # if os_is_mac(): # self.mount(device=device) user = os.environ.get('USER') script = textwrap.dedent(f""" ls /media/{user} sudo parted {device} --script -- mklabel msdos sudo parted {device} --script -- mkpart primary fat32 1MiB 100% sudo mkfs.vfat -n {_title} -F32 {device}1 sudo parted {device} --script print""").strip().splitlines() for line in script: _execute(line, line) os.system("sudo sync") if unmount: self.unmount(device=device ) # without dev we unmount but do not eject. If # we completely eject, burn will fail to detect the device. os.system("sudo sync") Console.ok("Formatted SD Card") elif os_is_mac(): details = USB.get_dev_from_diskutil() # checking if string contains list element valid = any(entry in device for entry in details) if not valid: Console.error( f"this device can not be used for formatting: {device}") return False elif len(details) > 1: Console.error( "For security reasons, please only put one USB writer in") Console.msg(f"we found {details}") return False else: details = USB.get_from_diskutil() USB.print_details(details) print() if yes or yn_choice( f"\nDo you like to format {device} as {_title}"): _execute( f"Formatting {device} as {_title}", f"sudo diskutil eraseDisk FAT32 {_title} MBRFormat {device}" ) else: raise NotImplementedError("Not implemented for this OS") return True
def do_burn(self, args, arguments): """ :: Usage: burn gui [--hostname=HOSTNAME] [--ip=IP] [--ssid=SSID] [--wifipassword=PSK] [--bs=BLOCKSIZE] [--dryrun] [--no_diagram] burn ubuntu NAMES [--inventory=INVENTORY] [--ssid=SSID] [-f] [--wifipassword=PSK] [-v] --device=DEVICE [--country=COUNTRY] [--upgrade] burn raspberry NAMES --device=DEVICE [--inventory=INVENTORY] [--ssid=SSID] [--wifipassword=PSK] [--country=COUNTRY] [--password=PASSWORD] [-v] [-f] burn firmware check burn firmware update burn install burn load --device=DEVICE burn format --device=DEVICE burn imager [TAG...] burn mount [--device=DEVICE] [--os=OS] burn unmount [--device=DEVICE] [--os=OS] burn network list [--ip=IP] [--used] burn network burn info [--device=DEVICE] burn image versions [--details] [--refresh] [--yaml] burn image ls burn image delete [--image=IMAGE] burn image get [--url=URL] [TAG...] burn backup [--device=DEVICE] [--to=DESTINATION] burn copy [--device=DEVICE] [--from=DESTINATION] burn shrink [--image=IMAGE] burn cluster --device=DEVICE --hostname=HOSTNAME [--burning=BURNING] [--ip=IP] [--ssid=SSID] [--wifipassword=PSK] [--bs=BLOCKSIZE] [--os=OS] [-y] [--imaged] [--set_passwd] burn create [--image=IMAGE] [--device=DEVICE] [--burning=BURNING] [--hostname=HOSTNAME] [--ip=IP] [--sshkey=KEY] [--blocksize=BLOCKSIZE] [--passwd=PASSWD] [--ssid=SSID] [--wifipassword=PSK] [--format] [--tag=TAG] [--inventory=INVENTORY] [--name=NAME] [-y] burn sdcard [TAG...] [--device=DEVICE] [-y] burn set [--hostname=HOSTNAME] [--ip=IP] [--key=KEY] [--keyboard=COUNTRY] [--cmdline=CMDLINE] burn enable ssh burn wifi --ssid=SSID [--passwd=PASSWD] [--country=COUNTRY] burn check [--device=DEVICE] burn mac --hostname=HOSTNAME Options: -h --help Show this screen. --version Show version. --image=IMAGE The image filename, e.g. 2019-09-26-raspbian-buster.img --device=DEVICE The device, e.g. /dev/sdX --hostname=HOSTNAME The hostnames of the cluster --ip=IP The IP addresses of the cluster --key=KEY The name of the SSH key file --blocksize=BLOCKSIZE The blocksise to burn [default: 4M] --burning=BURNING The hosts to be burned Arguments: TAG Keyword tags to identify an image Files: This is not fully thought through and needs to be documented ~/.cloudmesh/images Location where the images will be stored for reuse Description: cms burn create --inventory=INVENTORY --device=DEVICE --name=NAME Will refer to a specified cloudmesh inventory file (see cms help inventory). Will search the configurations for NAME inside of INVENTORY and will burn to DEVICE. Supports parameter expansion. cms burn create --passwd=PASSWD if the passwd flag is added the default password is queried from the commandline and added to all SDCards if the flag is omitted login via the password is disabled and only login via the sshkey is allowed Network cms burn network list Lists the ip addresses that are on the same network +------------+---------------+----------+-----------+ | Name | IP | Status | Latency | |------------+---------------+----------+-----------| | Router | 192.168.1.1 | up | 0.0092s | | iPhone | 192.168.1.4 | up | 0.061s | | red01 | 192.168.1.46 | up | 0.0077s | | laptop | 192.168.1.78 | up | 0.058s | | unkown | 192.168.1.126 | up | 0.14s | | red03 | 192.168.1.158 | up | 0.0037s | | red02 | 192.168.1.199 | up | 0.0046s | | red | 192.168.1.249 | up | 0.00021s | +------------+----------------+----------+-----------+ cms burn network list [--used] Lists the used ip addresses as a comma separated parameter list 192.168.50.1,192.168.50.4,... cms burn network address Lists the own network address +---------+----------------+----------------+ | Label | Local | Broadcast | |---------+----------------+----------------| | wlan0 | 192.168.1.12 | 192.168.1.255 | +---------+----------------+----------------+ cms burn firmware check Checks if the firmware on the Pi is up to date cms burn firmware update Checks and updates the firmware on the Pi cms burn install Installs a program to shrink img files. THis is useful, after you created a backup to make the backup smaller and allow faster burning in case of recovery This command is not supported on MacOS cms burn load --device=DEVICE Loads the sdcard into the USB drive. Thi sis similar to loading a cdrom drive. It s the opposite to eject cms burn format --device=DEVICE Formats the SDCard in the specified device. Be careful it is the correct device. cms burn info will help you to identifying it cms burn mount [--device=DEVICE] [--os=OS] Mounts the file systems available on the SDCard cms burn unmount [--device=DEVICE] [--os=OS] Unmounts the mounted file systems from the SDCard cms burn info [--device=DEVICE] Provides useful information about the SDCard cms burn image versions [--refresh] [--yaml] The images that you like to burn onto your SDCard can be cached locally with the image command. The available images for the PI can be found when using the --refresh option. If you do not specify it it reads a copy of the image list from our cache cms burn image ls Lists all downloaded images in our cache. You can download them with the cms burn image get command cms burn image delete [--image=IMAGE] Deletes the specified image. The name can be found with the image ls command cms burn image get [--url=URL] [TAG...] Downloads a specific image or the latest image. The tag are a number of words separated by a space that must occur in the tag that you find in the versions command cms burn backup [--device=DEVICE] [--to=DESTINATION] This command requires you to install pishrink previously with cms burn install Backs up a SDCard to the given location. cms burn copy [--device=DEVICE] [--from=DESTINATION] Copies the file form the destination on the SDCard this is the same as the SDCard command. we will in future remove one cms burn shrink [--image=IMAGE] Shrinks the size of a backup or image file that is on your local file system. It can only be used for .img files This command is not supported on MacOS. cms burn create [--image=IMAGE] [--device=DEVICE] [--hostname=HOSTNAME] [--ip=IP] [--sshkey=KEY] [--blocksize=BLOCKSIZE] [--passwd=PASSWD] [--ssid=SSID] [--wifipassword=PSK] [--format] This command not only can format the SDCard, but also initializes it with specific values cms burn sdcard [TAG...] [--device=DEVICE] this burns the sd card, see also copy and create cms burn set [--hostname=HOSTNAME] [--ip=IP] [--key=KEY] [--mount=MOUNTPOINT] [--keyboard=COUNTRY] [--cmdline=CMDLINE] Sets specific values on the sdcard after it has ben created with the create, copy or sdcard command a --ssh is missing from this command cms burn enable ssh [--mount=MOUNTPOINT] Enables the ssh server once it is booted cms burn wifi --ssid=SSID [--passwd=PASSWD] [--country=COUNTRY] Sets the wifi ssid and password after the card is created, copied, or the sdcard is used. The option country option expects an ISO 3166-1 two digit country code. The default is "US" and the option not required if suitable. See https://en.wikipedia.org/wiki/ISO_3166-1 for other countries. cms burn check [--device=DEVICE] Lists the parameters that were set with the set or create command Examples: ( \\ is not shown) > cms burn create --image=2019-09-26-raspbian-buster-lite > --device=/dev/mmcblk0 > --hostname=red[5-7] > --ip=192.168.1.[5-7] > --sshkey=id_rsa > cms burn image get latest > cms burn image get https://downloads.raspberrypi.org/ > raspbian_lite/images/ > raspbian_lite-2018-10-11/2018-10-09-raspbian-stretch-lite.zip > cms burn image delete 2019-09-26-raspbian-buster-lite """ map_parameters(arguments, "details", "refresh", "device", "dryrun", "burning", "hostname", "ip", "sshkey", "blocksize", "ssid", "url", "imaged", "key", "keyboard", "passwd", "wifipassword", "version", "to", "os", "country", "inventory", "name", "bs", "set_passwd", "cmdline", "upgrade", "no_diagram") # arguments.MOUNTPOINT = arguments["--mount"] arguments.FORMAT = arguments["--format"] arguments.FROM = arguments["--from"] arguments.IMAGE = arguments["--image"] arguments.output = "table" # hard code for now arguments.bs = arguments.bs or "4M" arguments.yes = arguments["-y"] if len(arguments.TAG) == 0: arguments.TAG = "latest" # VERBOSE(arguments) def execute(label, function): StopWatch.start(label) result = function StopWatch.stop(label) StopWatch.status(label, True) return result burner = Burner() sdcard = SDCard() if arguments.imager: arguments.TAG = arguments.TAG or ["latest-lite"] Console.msg(f"Tags: {arguments.TAG}") try: file = Imager.fetch(tag=arguments.TAG) except: # noqa: E722 pass try: Imager.launch(file=file) except Exception as e: Console.error( f"could not find image with the tag {arguments.TAG}\n\n{e}\n" ) return "" elif arguments.gui: from cloudmesh.burn.gui import Gui VERBOSE(arguments) g = Gui(hostname=arguments.hostname, ip=arguments.ip, dryrun=arguments.dryrun, no_diagram=arguments.no_diagram) g.run() return "" elif arguments.raspberry: banner(txt="RaspberryOS Burn", figlet=True) if arguments.inventory: inv_path = path_expand(f'~/.cloudmesh/{arguments.inventory}') try: burner = RaspberryBurner( inventory=inv_path, ssid=arguments['--ssid'], wifipassword=arguments['--wifipassword'], country=arguments['--country']) except: Console.error('Burner Error') return "" else: try: burner = RaspberryBurner( names=arguments.NAMES, ssid=arguments['--ssid'], wifipassword=arguments['--wifipassword'], force_inv=arguments['-f'], country=arguments['--country']) except Exception as e: Console.error('Burner Error') raise e return "" execute( "burn raspberry", burner.multi_burn( names=arguments.NAMES, devices=arguments.device, verbose=arguments['-v'], password=arguments['--password'], )) return "" elif arguments.ubuntu: banner(txt="Ubuntu Burn with cloud-init", figlet=True) names = Parameter.expand(arguments.NAMES) if len(Parameter.expand(arguments.device)) > 1: Console.error( "Too many devices specified. Please only specify one") return "" if arguments.inventory: c = Configure(inventory=arguments.inventory, debug=arguments['-v']) inv = Inventory(filename=arguments.inventory) else: names = Parameter.expand(arguments.NAMES) manager, workers = Host.get_hostnames(names) if workers: worker_base_name = ''.join( [i for i in workers[0] if not i.isdigit()]) cluster_name = manager or worker_base_name inventory = path_expand( f'~/.cloudmesh/inventory-{cluster_name}.yaml') if not os.path.exists(inventory) or arguments['-f']: if not manager: Console.error("No inventory found. Can not create an " "inventory without a " "manager.") return "" Inventory.build_default_inventory( filename=inventory, manager=manager, workers=workers, manager_image='ubuntu-20.10-64-bit', worker_image='ubuntu-20.10-64-bit') c = Configure(inventory=inventory, debug=arguments['-v'], download_images=True) inv = Inventory(filename=inventory) names = Parameter.expand(arguments.NAMES) manager, workers = Host.get_hostnames(names) if manager: if not arguments.ssid and 'wifi' in c.configs[manager][ 'services']: arguments.ssid = get_ssid() if arguments.ssid == "": Console.info('Could not determine SSID, skipping wifi ' 'config') arguments.ssid = None if not arguments.wifipassword and arguments.ssid is not None: arguments.country = Shell.locale().upper() arguments.wifipassword = getpass( f"Using --SSID=" f"{arguments.ssid} and " f" --COUNTRY=" f"{arguments.country}, please " f"enter wifi password:"******"" if 'ubuntu' not in tag: Console.error( "This command only supports burning ubuntu cards") return "" sdcard = SDCard(card_os="ubuntu") # Code below taken from arguments.sdcard try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" # determine if we are burning a manager, as this needs to be done # first to get the ssh public key # manager = False # for name in names: # if not inv.has_host(name): # Console.error(f'Could not find {name} in inventory {inv.filename}') # return "" # service = inv.get(name=name, attribute='service') # if service == 'manager' and not manager: # manager = name # # make manager first in names # names.remove(name) # names.insert(0, name) # elif service == 'manager' and manager: # raise Exception('More than one manager detected in NAMES') for name in names: if not yn_choice( f'Is the card to be burned for {name} inserted?'): if not yn_choice( f"Please insert the card to be burned for {name}. " "Type 'y' when done or 'n' to terminante"): Console.error("Terminating: User Break") return "" service = inv.get(name=name, attribute='service') # Make sure bridge is only enabled if WiFi enabled if service == 'manager': services = inv.get(name=name, attribute='services') if 'bridge' in services and not arguments.ssid: Console.error( 'Service bridge can only be configured if WiFi' ' is enabled with --ssid and --wifipassword') return "" else: enable_bridge = 'bridge' in services Console.info(f'Burning {name}') sdcard.format_device(device=arguments.device, yes=True) sdcard.unmount(device=arguments.device) sdcard.burn_sdcard(tag=tag, device=arguments.device, yes=True) sdcard.mount(device=arguments.device, card_os="ubuntu") if service == 'manager': # Generate a private public key pair for the manager that will be persistently used # priv_key, pub_key = c.generate_ssh_key(name) # Write priv_key and pub_key to /boot/id_rsa and /boot/id_rsa.pub # SDCard.writefile(filename=f'{sdcard.boot_volume}/id_rsa', content=priv_key) # SDCard.writefile(filename=f'{sdcard.boot_volume}/id_rsa.pub', content=pub_key) c.build_user_data( name=name, country=arguments.country, upgrade=arguments.upgrade, with_bridge=enable_bridge).write( filename=sdcard.boot_volume + '/user-data') c.build_network_data(name=name, ssid=arguments.ssid, password=arguments.wifipassword)\ .write(filename=sdcard.boot_volume + '/network-config') else: c.build_user_data( name=name, add_manager_key=manager, upgrade=arguments.upgrade).write( filename=sdcard.boot_volume + '/user-data') c.build_network_data(name=name).write( filename=sdcard.boot_volume + '/network-config') time.sleep( 1 ) # Sleep for 1 seconds to give ample time for writing to finish sdcard.unmount(device=arguments.device, card_os="ubuntu") Console.info("Remove card") Console.ok(f"Burned {len(names)} card(s)") return "" elif arguments.firmware and arguments.check: execute("firmware check", burner.firmware(action="check")) return "" elif arguments.firmware and arguments.update: execute("firmware update", burner.firmware(action="update")) return "" if arguments.check: execute("check", burner.check(device=arguments.device)) return "" elif arguments.versions and arguments['image']: StopWatch.start("image versions") result = Image.create_version_cache(refresh=arguments["--refresh"]) output = "table" if arguments["--yaml"]: output = "yaml" order = ["tag", 'date', "os", "type", 'version'] header = ["Tag", 'Date', "OS", "Type", 'Version'] if arguments.details: order = ["tag", 'date', "os", "type", 'version', "url"] header = ["Tag", 'Date', "OS", "Type", 'Version', "Url"] print( Printer.write(result, order=order, header=header, output=output)) StopWatch.stop("image versions") StopWatch.status("image versions", True) return "" elif arguments.load: execute("load", sdcard.load_device(device=arguments.device)) return "" elif arguments[ "format"]: # as format is a python word, we need to use an index execute( "format", sdcard.format_device(device=arguments.device, unmount=True)) return "" elif arguments.network and arguments["list"]: if os_is_mac(): Console.error("Not yet implemented on MacOS") return "" ip = arguments.ip or Network.address()[0]['local'] details = Network.nmap(ip=ip) if arguments.used: print(','.join([x['ip'] for x in details])) else: print( Printer.write(details, order=[ 'name', "ip", "status", "latency", ], header=[ 'Name', "IP", "Status", "Latency", ])) return "" elif arguments.network: if os_is_mac(): Console.error("Not yet implemented on MacOS") return "" # print (Network.nmap()) details = Network.address() print( Printer.write(details, order=['label', "local", "broadcast"], header=["Label", "Local", "Broadcast"])) return "" elif arguments.wifi: password = arguments.passwd ssid = arguments.ssid or get_ssid() country = arguments.country if password is None: password = getpass("Please enter the Wifi password or enter " "for no password: "******"macos" elif os_is_linux(): host = "linux" elif os_is_pi(): host = "raspberry" else: Console.error( "This command is not yet implemented for your OS") return "" burner.configure_wifi(ssid, psk=password, country=country, host=host) return "" elif arguments.info: output = arguments.output or "table" card = SDCard() execute("info", card.info(output=output)) try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" return "" elif arguments.install: if os_is_mac(): Console.error("Not yet implemented on MacOS") return "" execute("install", burner.install()) return "" elif arguments.shrink: if os_is_mac(): Console.error("Not yet implemented on MacOS") return "" execute("shrink", burner.shrink(image=arguments.IMAGE)) return "" elif arguments.backup: try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" execute( "backup", sdcard.backup(device=arguments.device, to_file=arguments.to)) return "" elif arguments[ "copy"]: # as copy is a reserved word we need to use the index USB.check_for_readers() execute( "copy", sdcard.copy(device=arguments.device, from_file=arguments.FROM)) return "" elif arguments.sdcard: try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" if arguments.device is None: card = SDCard() card.info() Console.error("Please specify a device") return "" arguments.TAG = arguments.TAG or ["latest-lite"] if any("ubuntu" in tag for tag in arguments.TAG): sdcard = SDCard(card_os="ubuntu") execute( "format", sdcard.format_device(device=arguments.device, unmount=True)) execute("unmount", sdcard.unmount(device=arguments.device)) execute( "sdcard", sdcard.burn_sdcard(tag=arguments.TAG, device=arguments.device, yes=arguments.yes)) return "" elif arguments.mount: if arguments.device is None: card = SDCard card.info() Console.error("Please specify a device") return "" execute( "mount", sdcard.mount(device=arguments.device, card_os=arguments.os)) return "" elif arguments.unmount: card = SDCard(card_os=arguments.os) execute( "unmount", card.unmount(device=arguments.device, card_os=arguments.os)) return "" elif arguments.mac: hostnames = Parameter.expand(arguments.hostname) execute("mac", burner.mac(hostnames=hostnames)) return "" elif arguments.set: try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" if arguments.hostname: execute("set hostname", burner.set_hostname(arguments.hostname)) if arguments.ip: execute("set ip", burner.set_static_ip(arguments.ip)) if arguments.key: execute("set key", burner.set_key(arguments.key)) if arguments.keyboard: execute("set keyboard", burner.keyboard(country=arguments.keyboard)) if arguments.cmdline: execute("set cmdline", burner.set_cmdline(arguments.cmdline)) return "" elif arguments.enable and arguments.ssh: try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" execute("enable ssh", burner.enable_ssh()) return "" # elif arguments.versions and arguments.image: # image = Image() elif arguments.ls and arguments['image']: execute("image ls", Image().ls()) return "" elif arguments.delete and arguments.IMAGE: execute("image rm", Image().rm(arguments.IMAGE)) return "" elif arguments["get"] and arguments['image'] and arguments["--url"]: image = Image() execute("image fetch", image.fetch(url=arguments.url)) return "" elif arguments["get"] and arguments['image'] and arguments["TAG"]: tag = arguments["TAG"] if "latest" in tag and ("full" in tag or "lite" in tag): result = Image.create_version_cache( refresh=arguments["--refresh"]) image = Image() execute("image fetch", image.fetch(tag=arguments["TAG"])) return "" elif arguments["get"] and arguments['image']: image = Image() execute("image fetch", image.fetch(tag="latest")) return "" elif arguments.cluster: # is true when # # cms burn cluster --hostname=red,red00[1-2] # --device=/dev/sdb # --ip=10.1.1.[1-3] # --ssid=myssid # --wifipassword=mypass # try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" execute("cluster", burner.cluster(arguments=arguments)) return "" elif arguments.create and arguments.inventory: try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" if not os_is_pi(): print() Console.error( "This command has only been written for a Raspberry Pis. " "Terminating for caution") print() if yn_choice("Continue anyways?"): pass else: return if not arguments.name: Console.error( "Missing --name parameter. See cms help burn for usage") return "" if not arguments.device: Console.error( "Missing --device parameter. See cms help burn for usage") return "" StopWatch.start("burn inventory") multi_burner = MultiBurner() # Perhaps we want to change the path at some point inventory = f"~/.cloudmesh/{arguments.inventory}" multi_burner.burn_inventory(inventory=inventory, name=arguments.name, device=arguments.device, yes=arguments.yes, passwd=arguments.passwd) StopWatch.stop("burn inventory") StopWatch.status("burn inventory", True) StopWatch.benchmark(sysinfo=False, csv=False) return "" elif arguments.create: try: USB.check_for_readers() except Exception as e: print() Console.error(e) print() return "" if arguments["--passwd"]: passwd = arguments["--passwd"] elif "PASSWD" in os.environ: passwd = os.environ["PASSWD"] else: passwd = generate_strong_pass() psk = None if arguments["--ssid"]: ssid = arguments["--ssid"] if arguments["--wifipassword"]: psk = arguments["--wifipassword"] else: psk = None else: if arguments["--wifipassword"]: print("Can't have wifi password with no ssid") return else: ssid = None image = 'latest' or arguments.IMAGE dev = os.environ['DEV'] if 'DEV' in os.environ else None devices = arguments["--device"] or dev or None if devices is not None: devices = Parameter.expand_string(devices) hostnames = Parameter.expand(arguments.hostname) if arguments.burnimg is None: burning = hostnames else: burning = arguments.burning VERBOSE(arguments) ips = None if not arguments.ip else Parameter.expand(arguments.ip) key = arguments.sshkey tag = arguments['--tag'] if os_is_pi() or os_is_linux(): blocksize = arguments.blocksize StopWatch.start("total") multi = MultiBurner() multi.burn_all( burning=burning, image=image, device=devices, blocksize=blocksize, progress=True, hostnames=hostnames, # not difference between names and name, maybe we should align ips=ips, key=key, password=passwd, ssid=ssid, psk=psk, tag=tag, yes=arguments.yes) StopWatch.stop("total") StopWatch.status("total", True) StopWatch.benchmark(sysinfo=False, csv=False) else: Console.error( "This command is only supported ona Pi and Linux") return "" Console.error("see manual page: cms help burn") return ""
def info(self, print_os=True, print_fdisk=True, print_stdout=True, output="table"): """ Finds out information about USB devices TODO: should we rename print_stdout to debug? seems more in line with cloudmesh :param print_os: :type print_os: :param print_fdisk: :type print_fdisk: :param print_stdout: if set to True prints debug information :type print_stdout: bool :param output: :type output: :return: dict with details about the devices :rtype: dict """ if print_os and print_stdout: if os_is_pi(): banner("This is Raspberry PI") elif os_is_mac(): banner("This is Mac") elif os_is_windows(): banner("This is a Windows Computer") elif os_is_linux(): banner("This is a Linux Computer") else: Console.error("unkown OS") sys.exit(1) if os_is_pi() and print_fdisk and print_stdout: result = USB.fdisk("/dev/mmcblk0") if print_stdout: banner("Operating System SD Card") print(result) details = USB.get_from_usb() if print_stdout and details is not None: banner("USB Device Probe") print( Printer.write(details, order=[ "address", "bus", "idVendor", "idProduct", "hVendor", "hProduct", "iManufacturer", "iSerialNumber", "usbVersion", "comment" ], header=[ "Adr.", "bus", "Vendor", "Prod.", "H Vendor", "H Prod.", "Man.", "Ser.Num.", "USB Ver.", "Comment" ], output=output)) # devices = USB.get_devices() # banner("Devices found") # print ('\n'.join(sorted(devices))) if os_is_mac(): names = USB.get_dev_from_diskutil() details = USB.get_from_diskutil() else: details = USB.get_from_dmesg() if print_stdout: banner("SD Cards Found") if os_is_mac(): print("We found the follwing cards:") print(" - /dev/" + "\n - /dev/".join(names)) print() print("We found the follong file systems on these disks:") print() print( Printer.write(details, order=[ "dev", "info", "formatted", "size", "active", "readable", "empty", "direct-access", "removable", "writeable" ], header=[ "Path", "Info", "Formatted", "Size", "Plugged-in", "Readable", "Empty", "Access", "Removable", "Writeable" ], output=output)) # lsusb = USB.get_from_lsusb() # from pprint import pprint # pprint (lsusb) # endors = USB.get_vendor() # print(vendors) # udev = subprocess.getoutput("udevadm info -a -p $(udevadm info -q path -n /dev/sda)") # # attributes = ["vendor","model", "model", "version", "manufacturer", # "idProduct", "idVendor"] # for line in udev.splitlines(): # if any(word in line for word in attributes): # print(line) if print_stdout: if os_is_linux(): card = SDCard() m = card.ls() banner("Mount points") if len(m) != 0: print( Printer.write(m, order=[ "name", "path", "type", "device", "parameters" ], header=[ "Name", "Path", "Type", "Device", "Parameters" ], output=output)) else: Console.warning( "No mount points found. Use cms burn mount") print() # Convert details into a dict where the key for each entry is the device details = {detail['dev']: detail for detail in details} return details
def burn_sdcard(self, image=None, tag=None, device=None, blocksize="4M", name="the inserted card", yes=False): """ Burns the SD Card with an image :param image: Image object to use for burning (used by copy) :type image: str :param name: :type name: str :param tag: tag object used for burning (used by sdcard) :type tag: str :param device: Device to burn to, e.g. /dev/sda :type device: str :param blocksize: the blocksize used when writing, default 4M :type blocksize: str :param yes: :type yes: str """ if image and tag: Console.error("Implementation error, burn_sdcard can't have image " "and tag.") return "" Console.info(f"Burning {name} ...") if image is not None: image_path = image else: image = Image().find(tag=tag) if image is None: Console.error("No matching image found.") return "" elif len(image) > 1: Console.error("Too many images found") print( Printer.write(image, order=["tag", "version"], header=["Tag", "Version"])) return "" image = image[0] if "ubuntu" in image["url"]: _name = os.path.basename(Image.get_name(image["url"])) _name = _name.replace(".xz", "") else: _name = os.path.basename(Image.get_name(image["url"])) + ".img" image_path = Image().directory + "/" + _name print(image_path) if not os.path.isfile(image_path): tags = ' '.join(tag) print() Console.error( f"Image with tags '{tags}' not found. To download use") print() Console.msg(f"cms burn image get {tags}") print() return "" orig_size = size = humanize.naturalsize(os.path.getsize(image_path)) # size = details[0]['size'] n, unit = size.split(" ") unit = unit.replace("GB", "G") unit = unit.replace("MB", "M") n = float(n) if unit == "G": n = n * 1000**3 elif unit == "M": n = n * 1000**2 size = int(n) banner(f"Preparing the SDCard {name}") print(f"Name: {name}") print(f"Image: {image_path}") print(f"Image Size: {orig_size}") print(f"Device: {device}") print(f"Blocksize: {blocksize}") if os_is_mac(): blocksize = blocksize.lower() print() Sudo.password() if device is None: Console.error("Please specify a device") return # # speedup burn for MacOS # if device.startswith("/dev/disk"): device = device.replace("/dev/disk", "/dev/rdisk") if os_is_mac(): details = USB.get_from_diskutil() USB.print_details(details) if not (yes or yn_choice(f"\nDo you like to write {name} on {device} " f"with the image {image_path}")): return "" # TODO Gregor verify this is ok commenting out this line # self.mount(device=device) if os_is_mac(): command = f"sudo dd if={image_path} bs={blocksize} |" \ f' tqdm --bytes --total {size} --ncols 80 |' \ f" sudo dd of={device} bs={blocksize}" else: # command = f"sudo dd if={image_path} of={device} bs={blocksize} status=progress conv=fsync" command = f"sudo dd if={image_path} bs={blocksize} oflag=direct |" \ f' tqdm --bytes --total {size} --ncols 80 |' \ f" sudo dd of={device} bs={blocksize} iflag=fullblock " \ f"oflag=direct conv=fsync" print(command) os.system(command) Sudo.execute("sync") if os_is_linux(): self.unmount(device=device, full=True) else: self.unmount(device=device)
def mount(self, device=None, card_os="raspberry"): """ Mounts the current SD card """ if os_is_linux(): Sudo.password() dmesg = USB.get_from_dmesg() # TODO Need a better way to identify which sd card to use for mounting # instead of iterating over all of them for usbcard in dmesg: dev = device or usbcard['dev'] print(f"Mounting filesystems on {dev}") try: Console.ok(f"mounting {device}") os.system( 'sudo sync') # flush any pending/in-process writes os.system(f"sudo eject {device}") os.system(f"sudo eject -t {device}") os.system( 'sudo sync') # flush any pending/in-process writes # ensure the card is mounted before returning device_basename = os.path.basename(device) part1 = False part2 = False for i in range(20): result = Shell.run('lsblk') if device_basename in result.split(): for line in result.splitlines(): line = line.split() if device_basename + '1' in line[0] and len( line) > 6: part1 = True elif device_basename + '2' in line[0] and len( line) > 6: part2 = True if part1 and part2: # card is fully mounted break time.sleep(0.5) if not part1 and not part2: raise Exception("card failed to mount both partitions") except Exception as e: print(e) elif os_is_pi(): Sudo.password() if card_os is None: Console.error("Please specify the OS you have on the SD Card") return "" self.card_os = card_os dmesg = USB.get_from_dmesg() print(dmesg) # TODO Need a better way to identify which sd card to use for mounting # instead of iterating over all of them os.system('sudo sync') # flush any pending/in-process writes for usbcard in dmesg: dev = device or usbcard['dev'] print( f"Mounting filesystems on {dev} assuming it is {card_os} as you specified" ) sd1 = f"{dev}1" sd2 = f"{dev}2" try: if os.path.exists(sd1): Console.ok(f"mounting {sd1} {self.boot_volume}") os.system(f"sudo mkdir -p {self.boot_volume}") os.system( f"sudo mount -t vfat {sd1} {self.boot_volume}") except Exception as e: print(e) try: if os.path.exists(sd2): Console.ok(f"mounting {sd2} {self.root_volume}") os.system(f"sudo mkdir -p {self.root_volume}") os.system( f"sudo mount -t ext4 {sd2} {self.root_volume}") except Exception as e: print(e) elif os_is_mac(): command = f"diskutil mountDisk {device}" print(command) os.system(command) else: Console.error("Not yet implemented for your OS") Sudo.execute("sync") return ""
from cloudmesh.burn.sdcard import SDCard from cloudmesh.burn.util import os_is_mac from cloudmesh.burn.util import os_is_linux from cloudmesh.burn.util import os_is_pi from cloudmesh.common.Benchmark import Benchmark from cloudmesh.common.Shell import Shell from cloudmesh.common.console import Console from cloudmesh.common.systeminfo import get_platform from cloudmesh.common.util import HEADING from cloudmesh.common.util import yn_choice from cloudmesh.common.util import banner from cloudmesh.burn.usb import USB cloud = get_platform() if os_is_mac(): device = "/dev/disk2" details = USB.get_dev_from_diskutil() # checking if string contains list element valid = any(entry in device for entry in details) if not valid: Console.error("this device can not be used for the test") sys.exit(1) elif len(details) > 1: Console.error( "For security reasons, please only put one USB writer in") sys.exit(1)
[File Selector] [x] Format Manager: _______________ Workers: _______________ [Burn] [Cancel] """ logo = './cm-logo.png' # TODO: we do teh device wrong for now as we want to show radio button if os_is_mac(): details = USB.get_from_diskutil() else: details = USB.get_from_dmesg() devices = yaml.safe_load(Printer.write(details, order=[ "dev", "info", "formatted", "size", "active", "readable", "empty", "direct-access", "removable",