def scan_disks(min_size): root = root_disk() cmd = [ '/usr/bin/lsblk', '-P', '-o', 'NAME,MODEL,SERIAL,SIZE,TRAN,VENDOR,HCTL,TYPE,FSTYPE,LABEL,UUID' ] o, e, rc = run_command(cmd) dnames = {} disks = [] serials = [] root_serial = None # to use udevadm to retrieve serial number rather than lsblk, make this True always_use_udev_serial = False device_names_seen = [] for l in o: if (re.match('NAME', l) is None): continue dmap = {} cur_name = '' cur_val = '' name_iter = True val_iter = False sl = l.strip() i = 0 while i < len(sl): if (name_iter and sl[i] == '=' and sl[i + 1] == '"'): name_iter = False val_iter = True i = i + 2 elif (val_iter and sl[i] == '"' and (i == (len(sl) - 1) or sl[i + 1] == ' ')): val_iter = False name_iter = True i = i + 2 dmap[cur_name.strip()] = cur_val.strip() cur_name = '' cur_val = '' elif (name_iter): cur_name = cur_name + sl[i] i = i + 1 elif (val_iter): cur_val = cur_val + sl[i] i = i + 1 else: raise Exception('Failed to parse lsblk output: %s' % sl) # md devices, such as mdadmin software raid and some hardware raid block # devices show up in lsblk's output multiple times with identical info. # Given we only need one copy of this info we remove duplicate device # name entries, also offers more sane output to views/disk.py where name # will be used as the index if (dmap['NAME'] in device_names_seen): continue device_names_seen.append(dmap['NAME']) # We are not interested in CD / DVD rom devices so skip to next device if (dmap['TYPE'] == 'rom'): continue if (dmap['NAME'] == root): root_serial = dmap['SERIAL'] if (dmap['TYPE'] == 'part'): for dname in dnames.keys(): if (re.match(dname, dmap['NAME']) is not None): dnames[dname][11] = True if (((dmap['NAME'] != root and dmap['TYPE'] != 'part') or (dmap['TYPE'] == 'part' and dmap['FSTYPE'] == 'btrfs'))): dmap['parted'] = False # part = False by default dmap['root'] = False if (dmap['TYPE'] == 'part' and dmap['FSTYPE'] == 'btrfs'): # btrfs partition for root (rockstor_rockstor) pool if (re.match(root, dmap['NAME']) is not None): dmap['SERIAL'] = root_serial dmap['root'] = True else: # ignore btrfs partitions that are not for rootfs. continue # convert size into KB size_str = dmap['SIZE'] if (size_str[-1] == 'G'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024) elif (size_str[-1] == 'T'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024 * 1024) else: continue if (dmap['SIZE'] < min_size): continue if (dmap['SERIAL'] == '' or always_use_udev_serial): # lsblk fails to retrieve SERIAL from VirtIO drives and some # sdcard devices so try specialized function. # dmap['SERIAL'] = get_virtio_disk_serial(dmap['NAME']) dmap['SERIAL'] = get_disk_serial(dmap['NAME'], None) if (dmap['SERIAL'] == '' or (dmap['SERIAL'] in serials)): # No serial number still or its a repeat. # Overwrite drive serial entry in db with 'fake-serial-' + uuid4 # see disk/disks_table.jst for a use of this flag mechanism. # Previously we did dmap['SERIAL'] = dmap['NAME'] which is less # robust as it can itself produce duplicate serial numbers in db dmap['SERIAL'] = 'fake-serial-' + str(uuid.uuid4()) # 12 chars (fake-serial-) + 36 chars (uuid4) = 48 chars serials.append(dmap['SERIAL']) for k in dmap.keys(): if (dmap[k] == ''): dmap[k] = None dnames[dmap['NAME']] = [ dmap['NAME'], dmap['MODEL'], dmap['SERIAL'], dmap['SIZE'], dmap['TRAN'], dmap['VENDOR'], dmap['HCTL'], dmap['TYPE'], dmap['FSTYPE'], dmap['LABEL'], dmap['UUID'], dmap['parted'], dmap['root'], ] for d in dnames.keys(): disks.append(Disk(*dnames[d])) return disks
def scan_disks(min_size): root = root_disk() cmd = ['/usr/bin/lsblk', '-P', '-o', 'NAME,MODEL,SERIAL,SIZE,TRAN,VENDOR,HCTL,TYPE,FSTYPE,LABEL,UUID'] o, e, rc = run_command(cmd) dnames = {} disks = [] serials = [] root_serial = None # to use udevadm to retrieve serial number rather than lsblk, make this True always_use_udev_serial = False device_names_seen = [] for l in o: if (re.match('NAME', l) is None): continue dmap = {} cur_name = '' cur_val = '' name_iter = True val_iter = False sl = l.strip() i = 0 while i < len(sl): if (name_iter and sl[i] == '=' and sl[i + 1] == '"'): name_iter = False val_iter = True i = i + 2 elif (val_iter and sl[i] == '"' and (i == (len(sl) - 1) or sl[i + 1] == ' ')): val_iter = False name_iter = True i = i + 2 dmap[cur_name.strip()] = cur_val.strip() cur_name = '' cur_val = '' elif (name_iter): cur_name = cur_name + sl[i] i = i + 1 elif (val_iter): cur_val = cur_val + sl[i] i = i + 1 else: raise Exception('Failed to parse lsblk output: %s' % sl) # md devices, such as mdadmin software raid and some hardware raid block # devices show up in lsblk's output multiple times with identical info. # Given we only need one copy of this info we remove duplicate device # name entries, also offers more sane output to views/disk.py where name # will be used as the index if (dmap['NAME'] in device_names_seen): continue device_names_seen.append(dmap['NAME']) # We are not interested in CD / DVD rom devices so skip to next device if (dmap['TYPE'] == 'rom'): continue if (dmap['NAME'] == root): root_serial = dmap['SERIAL'] if (dmap['TYPE'] == 'part'): for dname in dnames.keys(): if (re.match(dname, dmap['NAME']) is not None): dnames[dname][11] = True if (((dmap['NAME'] != root and dmap['TYPE'] != 'part') or (dmap['TYPE'] == 'part' and dmap['FSTYPE'] == 'btrfs'))): dmap['parted'] = False # part = False by default dmap['root'] = False if (dmap['TYPE'] == 'part' and dmap['FSTYPE'] == 'btrfs'): # btrfs partition for root (rockstor_rockstor) pool if (re.match(root, dmap['NAME']) is not None): dmap['SERIAL'] = root_serial dmap['root'] = True else: # ignore btrfs partitions that are not for rootfs. continue # convert size into KB size_str = dmap['SIZE'] if (size_str[-1] == 'G'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024) elif (size_str[-1] == 'T'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024 * 1024) else: continue if (dmap['SIZE'] < min_size): continue if (dmap['SERIAL'] == '' or always_use_udev_serial): # lsblk fails to retrieve SERIAL from VirtIO drives and some # sdcard devices so try specialized function. # dmap['SERIAL'] = get_virtio_disk_serial(dmap['NAME']) dmap['SERIAL'] = get_disk_serial(dmap['NAME'], None) if (dmap['SERIAL'] == '' or (dmap['SERIAL'] in serials)): # No serial number still or its a repeat. # Overwrite drive serial entry in db with 'fake-serial-' + uuid4 # see disk/disks_table.jst for a use of this flag mechanism. # Previously we did dmap['SERIAL'] = dmap['NAME'] which is less # robust as it can itself produce duplicate serial numbers in db dmap['SERIAL'] = 'fake-serial-' + str(uuid.uuid4()) # 12 chars (fake-serial-) + 36 chars (uuid4) = 48 chars serials.append(dmap['SERIAL']) for k in dmap.keys(): if (dmap[k] == ''): dmap[k] = None dnames[dmap['NAME']] = [dmap['NAME'], dmap['MODEL'], dmap['SERIAL'], dmap['SIZE'], dmap['TRAN'], dmap['VENDOR'], dmap['HCTL'], dmap['TYPE'], dmap['FSTYPE'], dmap['LABEL'], dmap['UUID'], dmap['parted'], dmap['root'], ] for d in dnames.keys(): disks.append(Disk(*dnames[d])) return disks
def scan_disks(min_size): """ Using lsblk we scan all attached disks and categorize them according to if they are partitioned, their file system, if the drive hosts our / mount point etc. The result of this scan is used by:- view/disk.py _update_disk_state for further analysis / categorization. N.B. if a device (partition or whole dev) hosts swap or is of no interest then it is ignored. :param min_size: Discount all devices below this size in KB :return: List containing drives of interest """ # todo candidate for move to system/osi as not btrfs related base_root_disk = root_disk() cmd = [ '/usr/bin/lsblk', '-P', '-o', 'NAME,MODEL,SERIAL,SIZE,TRAN,VENDOR,HCTL,TYPE,FSTYPE,LABEL,UUID' ] o, e, rc = run_command(cmd) dnames = {} # Working dictionary of devices. disks = [] # List derived from the final working dictionary of devices. serials_seen = [] # List tally of serials seen during this scan. # Stash variables to pass base info on root_disk to root device proper. root_serial = root_model = root_transport = root_vendor = root_hctl = None # To use udevadm to retrieve serial number rather than lsblk, make this True # N.B. when lsblk returns no serial for a device then udev is used anyway. always_use_udev_serial = False device_names_seen = [] # List tally of devices seen during this scan for line in o: # skip processing of all lines that don't begin with "NAME" if (re.match('NAME', line) is None): continue # setup our line / dev name dependant variables # easy read categorization flags, all False until found otherwise. is_root_disk = False # the base dev that / is mounted on ie system disk is_partition = is_btrfs = False dmap = { } # dictionary to hold line info from lsblk output eg NAME: sda # line parser variables cur_name = '' cur_val = '' name_iter = True val_iter = False sl = line.strip() i = 0 while i < len(sl): # We iterate over the line to parse it's information char by char # keeping track of name or value and adding the char accordingly if (name_iter and sl[i] == '=' and sl[i + 1] == '"'): name_iter = False val_iter = True i = i + 2 elif (val_iter and sl[i] == '"' and (i == (len(sl) - 1) or sl[i + 1] == ' ')): val_iter = False name_iter = True i = i + 2 dmap[cur_name.strip()] = cur_val.strip() cur_name = '' cur_val = '' elif (name_iter): cur_name = cur_name + sl[i] i = i + 1 elif (val_iter): cur_val = cur_val + sl[i] i = i + 1 else: raise Exception('Failed to parse lsblk output: %s' % sl) # md devices, such as mdadmin software raid and some hardware raid block # devices show up in lsblk's output multiple times with identical info. # Given we only need one copy of this info we remove duplicate device # name entries, also offers more sane output to views/disk.py where name # will be used as the index if (dmap['NAME'] in device_names_seen): continue device_names_seen.append(dmap['NAME']) # We are not interested in CD / DVD rom devices so skip to next device if (dmap['TYPE'] == 'rom'): continue # We are not interested in swap partitions or devices so skip further # processing and move to next device. # N.B. this also facilitates a simpler mechanism of classification. if (dmap['FSTYPE'] == 'swap'): continue # ----- Now we are done with easy exclusions we begin classification. # ------------ Start more complex classification ------------- if (dmap['NAME'] == base_root_disk): # as returned by root_disk() # We are looking at the system drive that hosts, either # directly or as a partition, the / mount point. # Given lsblk doesn't return serial, model, transport, vendor, hctl # when displaying partitions we grab and stash them while we are # looking at the root drive directly, rather than the / partition. # N.B. assumption is lsblk first displays devices then partitions, # this is the observed behaviour so far. root_serial = dmap['SERIAL'] root_model = dmap['MODEL'] root_transport = dmap['TRAN'] root_vendor = dmap['VENDOR'] root_hctl = dmap['HCTL'] # Set readability flag as base_dev identified. is_root_disk = True # root as returned by root_disk() # And until we find a partition on this root disk we will label it # as our root, this then allows for non partitioned root devices # such as mdraid installs where root is directly on eg /dev/md126. # N.B. this assumes base devs are listed before their partitions. dmap['root'] = True # Normal partitions are of type 'part', md partitions are of type 'md' # normal disks are of type 'disk' md devices are of type eg 'raid1'. # Disk members of eg intel bios raid md devices fstype='isw_raid_member' # Note for future re-write; when using udevadm DEVTYPE, partition and # disk works for both raid and non raid partitions and devices. # Begin readability variables assignment # - is this a partition regular or md type. if (dmap['TYPE'] == 'part' or dmap['TYPE'] == 'md'): is_partition = True # - is filesystem of type btrfs if (dmap['FSTYPE'] == 'btrfs'): is_btrfs = True # End readability variables assignment if is_partition: # Search our working dictionary of already scanned devices by name # We are assuming base devices are listed first and if of interest # we have recorded it and can now back port it's partitioned status. for dname in dnames.keys(): if (re.match(dname, dmap['NAME']) is not None): # Our device name has a base device entry of interest saved: # ie we have scanned and saved sdb but looking at sdb3 now. # Given we have found a partition on an existing base dev # we should update that base dev's entry in dnames to # parted "True" as when recorded lsblk type on base device # would have been disk or RAID1 or raid1 (for base md dev). # Change the 12th entry (0 indexed) of this device to True # The 12 entry is the parted flag so we label # our existing base dev entry as parted ie partitioned. dnames[dname][11] = True # Also take this opportunity to back port software raid info # from partitions to the base device if the base device # doesn't already have an fstype identifying it's raid # member status. For Example:- # bios raid base dev gives lsblk FSTYPE="isw_raid_member"; # we already catch this directly. # Pure software mdraid base dev has lsblk FSTYPE="" but a # partition on this pure software mdraid that is a member # of eg md125 has FSTYPE="linux_raid_member" if dmap['FSTYPE'] == 'linux_raid_member' \ and (dnames[dname][8] is None): # N.B. 9th item (index 8) in dname = FSTYPE # We are a partition that is an mdraid raid member so # backport this info to our base device ie sda1 raid # member so label sda's FSTYPE entry the same as it's # partition's entry if the above condition is met, ie # only if the base device doesn't already have an # FSTYPE entry ie None, this way we don't overwrite # / loose info and we only need to have one partition # identified as an mdraid member to classify the entire # device (the base device) as a raid member, at least in # part. dnames[dname][8] = dmap['FSTYPE'] if ((not is_root_disk and not is_partition) or (is_btrfs)): # We have a non system disk that is not a partition # or # We have a device that is btrfs formatted # In the case of a btrfs partition we override the parted flag. # Or we may just be a non system disk without partitions. dmap['parted'] = False # could be corrected later dmap[ 'root'] = False # until we establish otherwise as we might be. if is_btrfs: # a btrfs file system if (re.match(base_root_disk, dmap['NAME']) is not None): # We are assuming that a partition with a btrfs fs on is our # root if it's name begins with our base system disk name. # Now add the properties we stashed when looking at the base # root disk rather than the root partition we see here. dmap['SERIAL'] = root_serial dmap['MODEL'] = root_model dmap['TRAN'] = root_transport dmap['VENDOR'] = root_vendor dmap['HCTL'] = root_hctl # As we have found root to be on a partition we can now # un flag the base device as having been root prior to # finding this partition on that base_root_disk # N.B. Assumes base dev is listed before it's partitions # The 13th item in dnames entries is root so index = 12. # Only update our base_root_disk if it exists in our scanned # disks as this may be the first time we are seeing it. # Search to see if we already have an entry for the # the base_root_disk which may be us or our base dev if we # are a partition for dname in dnames.keys(): if dname == base_root_disk: dnames[base_root_disk][12] = False # And update this device as real root # Note we may be looking at the base_root_disk or one of # it's partitions there after. dmap['root'] = True # If we are an md device then use get_md_members string # to populate our MODEL since it is otherwise unused. if (re.match('md', dmap['NAME']) is not None): # cheap way to display our member drives dmap['MODEL'] = get_md_members(dmap['NAME']) else: # We have a non system disk btrfs filesystem. # Ie we are a whole disk or a partition with btrfs on but # NOT on the system disk. # Most likely a current btrfs data drive or one we could # import. # As we don't understand / support btrfs in partitions # then ignore / skip this btrfs device if it's a partition if is_partition: continue # convert size into KB size_str = dmap['SIZE'] if (size_str[-1] == 'G'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024) elif (size_str[-1] == 'T'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024 * 1024) else: # Move to next line if we don't understand the size as GB or TB # Note that this may cause an entry to be ignored if formatting # changes. # Previous to the explicit ignore swap clause this often caught # swap but if swap was in GB and above min_size then it could # show up when not in a partition (the previous caveat clause). continue if (dmap['SIZE'] < min_size): continue # No more continues so the device we have is to be passed to our db # entry system views/disk.py ie _update_disk_state() # Do final tidy of data in dmap and ready for entry in dnames dict. # db needs unique serial so provide one where there is none found. # First try harder with udev if lsblk failed on serial retrieval. if (dmap['SERIAL'] == '' or always_use_udev_serial): # lsblk fails to retrieve SERIAL from VirtIO drives and some # sdcard devices and md devices so try specialized function. dmap['SERIAL'] = get_disk_serial(dmap['NAME']) if (dmap['SERIAL'] == '' or (dmap['SERIAL'] in serials_seen)): # No serial number still or its a repeat. # Overwrite drive serial entry in dmap with fake-serial- + uuid4 # See disk/disks_table.jst for a use of this flag mechanism. # Previously we did dmap['SERIAL'] = dmap['NAME'] which is less # robust as it can itself produce duplicate serial numbers. dmap['SERIAL'] = 'fake-serial-' + str(uuid.uuid4()) # 12 chars (fake-serial-) + 36 chars (uuid4) = 48 chars serials_seen.append(dmap['SERIAL']) # replace all dmap values of '' with None. for key in dmap.keys(): if (dmap[key] == ''): dmap[key] = None # transfer our device info as now parsed in dmap to the dnames dict dnames[dmap['NAME']] = [ dmap['NAME'], dmap['MODEL'], dmap['SERIAL'], dmap['SIZE'], dmap['TRAN'], dmap['VENDOR'], dmap['HCTL'], dmap['TYPE'], dmap['FSTYPE'], dmap['LABEL'], dmap['UUID'], dmap['parted'], dmap['root'], ] # Transfer our collected disk / dev entries of interest to the disks list. for d in dnames.keys(): disks.append(Disk(*dnames[d])) return disks
def scan_disks(min_size): root = root_disk() cmd = [ '/usr/bin/lsblk', '-P', '-o', 'NAME,MODEL,SERIAL,SIZE,TRAN,VENDOR,HCTL,TYPE,FSTYPE,LABEL,UUID' ] o, e, rc = run_command(cmd) dnames = {} disks = [] serials = [] root_serial = None # to use udevadm for serial # rather than lsblk make this True always_use_udev_serial = False for l in o: if (re.match('NAME', l) is None): continue dmap = {} cur_name = '' cur_val = '' name_iter = True val_iter = False sl = l.strip() i = 0 while i < len(sl): if (name_iter and sl[i] == '=' and sl[i + 1] == '"'): name_iter = False val_iter = True i = i + 2 elif (val_iter and sl[i] == '"' and (i == (len(sl) - 1) or sl[i + 1] == ' ')): val_iter = False name_iter = True i = i + 2 dmap[cur_name.strip()] = cur_val.strip() cur_name = '' cur_val = '' elif (name_iter): cur_name = cur_name + sl[i] i = i + 1 elif (val_iter): cur_val = cur_val + sl[i] i = i + 1 else: raise Exception('Failed to parse lsblk output: %s' % sl) if (dmap['TYPE'] == 'rom'): continue if (dmap['NAME'] == root): root_serial = dmap['SERIAL'] if (dmap['TYPE'] == 'part'): for dname in dnames.keys(): if (re.match(dname, dmap['NAME']) is not None): dnames[dname][11] = True if (((dmap['NAME'] != root and dmap['TYPE'] != 'part') or (dmap['TYPE'] == 'part' and dmap['FSTYPE'] == 'btrfs'))): dmap['parted'] = False # part = False by default dmap['root'] = False if (dmap['TYPE'] == 'part' and dmap['FSTYPE'] == 'btrfs'): # btrfs partition for root(rockstor_rockstor) pool if (re.match(root, dmap['NAME']) is not None): dmap['SERIAL'] = root_serial dmap['root'] = True else: # ignore btrfs partitions that are not for rootfs. continue # convert size into KB size_str = dmap['SIZE'] if (size_str[-1] == 'G'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024) elif (size_str[-1] == 'T'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024 * 1024) else: continue if (dmap['SIZE'] < min_size): continue if (dmap['SERIAL'] == '' or always_use_udev_serial): # lsblk fails to retrieve SERIAL from VirtIO drives and some # sdcard devices so try specialized function. # dmap['SERIAL'] = get_virtio_disk_serial(dmap['NAME']) dmap['SERIAL'] = get_disk_serial(dmap['NAME'], None) if (dmap['SERIAL'] == '' or (dmap['SERIAL'] in serials)): # No serial number still or its a repeat. # Overwrite drive serial entry with drive name: # see disks_table.jst for a use of this flag mechanism. dmap['SERIAL'] = dmap['NAME'] serials.append(dmap['SERIAL']) for k in dmap.keys(): if (dmap[k] == ''): dmap[k] = None dnames[dmap['NAME']] = [ dmap['NAME'], dmap['MODEL'], dmap['SERIAL'], dmap['SIZE'], dmap['TRAN'], dmap['VENDOR'], dmap['HCTL'], dmap['TYPE'], dmap['FSTYPE'], dmap['LABEL'], dmap['UUID'], dmap['parted'], dmap['root'], ] for d in dnames.keys(): disks.append(Disk(*dnames[d])) return disks
def scan_disks(min_size): """ Using lsblk we scan all attached disks and categorize them according to if they are partitioned, their file system, if the drive hosts our / mount point etc. The result of this scan is used by:- view/disk.py _update_disk_state for further analysis / categorization. N.B. if a device (partition or whole dev) hosts swap or is of no interest then it is ignored. :param min_size: Discount all devices below this size in KB :return: List containing drives of interest """ base_root_disk = root_disk() cmd = ['/usr/bin/lsblk', '-P', '-o', 'NAME,MODEL,SERIAL,SIZE,TRAN,VENDOR,HCTL,TYPE,FSTYPE,LABEL,UUID'] o, e, rc = run_command(cmd) dnames = {} # Working dictionary of devices. disks = [] # List derived from the final working dictionary of devices. serials_seen = [] # List tally of serials seen during this scan. # Stash variables to pass base info on root_disk to root device proper. root_serial = root_model = root_transport = root_vendor = root_hctl = None # To use udevadm to retrieve serial number rather than lsblk, make this True # N.B. when lsblk returns no serial for a device then udev is used anyway. always_use_udev_serial = False device_names_seen = [] # List tally of devices seen during this scan for line in o: # skip processing of all lines that don't begin with "NAME" if (re.match('NAME', line) is None): continue # setup our line / dev name dependant variables # easy read categorization flags, all False until found otherwise. is_root_disk = False # the base dev that / is mounted on ie system disk is_partition = is_btrfs = False dmap = {} # dictionary to hold line info from lsblk output eg NAME: sda # line parser variables cur_name = '' cur_val = '' name_iter = True val_iter = False sl = line.strip() i = 0 while i < len(sl): # We iterate over the line to parse it's information char by char # keeping track of name or value and adding the char accordingly if (name_iter and sl[i] == '=' and sl[i + 1] == '"'): name_iter = False val_iter = True i = i + 2 elif (val_iter and sl[i] == '"' and (i == (len(sl) - 1) or sl[i + 1] == ' ')): val_iter = False name_iter = True i = i + 2 dmap[cur_name.strip()] = cur_val.strip() cur_name = '' cur_val = '' elif (name_iter): cur_name = cur_name + sl[i] i = i + 1 elif (val_iter): cur_val = cur_val + sl[i] i = i + 1 else: raise Exception('Failed to parse lsblk output: %s' % sl) # md devices, such as mdadmin software raid and some hardware raid block # devices show up in lsblk's output multiple times with identical info. # Given we only need one copy of this info we remove duplicate device # name entries, also offers more sane output to views/disk.py where name # will be used as the index if (dmap['NAME'] in device_names_seen): continue device_names_seen.append(dmap['NAME']) # We are not interested in CD / DVD rom devices so skip to next device if (dmap['TYPE'] == 'rom'): continue # We are not interested in swap partitions or devices so skip further # processing and move to next device. # N.B. this also facilitates a simpler mechanism of classification. if (dmap['FSTYPE'] == 'swap'): continue # ----- Now we are done with easy exclusions we begin classification. # ------------ Start more complex classification ------------- if (dmap['NAME'] == base_root_disk): # as returned by root_disk() # We are looking at the system drive that hosts, either # directly or as a partition, the / mount point. # Given lsblk doesn't return serial, model, transport, vendor, hctl # when displaying partitions we grab and stash them while we are # looking at the root drive directly, rather than the / partition. # N.B. assumption is lsblk first displays devices then partitions, # this is the observed behaviour so far. root_serial = dmap['SERIAL'] root_model = dmap['MODEL'] root_transport = dmap['TRAN'] root_vendor = dmap['VENDOR'] root_hctl = dmap['HCTL'] # Set readability flag as base_dev identified. is_root_disk = True # root as returned by root_disk() # And until we find a partition on this root disk we will label it # as our root, this then allows for non partitioned root devices # such as mdraid installs where root is directly on eg /dev/md126. # N.B. this assumes base devs are listed before their partitions. dmap['root'] = True # Normal partitions are of type 'part', md partitions are of type 'md' # normal disks are of type 'disk' md devices are of type eg 'raid1'. # Disk members of eg intel bios raid md devices fstype='isw_raid_member' # Note for future re-write; when using udevadm DEVTYPE, partition and # disk works for both raid and non raid partitions and devices. # Begin readability variables assignment # - is this a partition regular or md type. if (dmap['TYPE'] == 'part' or dmap['TYPE'] == 'md'): is_partition = True # - is filesystem of type btrfs if (dmap['FSTYPE'] == 'btrfs'): is_btrfs = True # End readability variables assignment if is_partition: # Search our working dictionary of already scanned devices by name # We are assuming base devices are listed first and if of interest # we have recorded it and can now back port it's partitioned status. for dname in dnames.keys(): if (re.match(dname, dmap['NAME']) is not None): # Our device name has a base device entry of interest saved: # ie we have scanned and saved sdb but looking at sdb3 now. # Given we have found a partition on an existing base dev # we should update that base dev's entry in dnames to # parted "True" as when recorded lsblk type on base device # would have been disk or RAID1 or raid1 (for base md dev). # Change the 12th entry (0 indexed) of this device to True # The 12 entry is the parted flag so we label # our existing base dev entry as parted ie partitioned. dnames[dname][11] = True # Also take this opportunity to back port software raid info # from partitions to the base device if the base device # doesn't already have an fstype identifying it's raid # member status. For Example:- # bios raid base dev gives lsblk FSTYPE="isw_raid_member"; # we already catch this directly. # Pure software mdraid base dev has lsblk FSTYPE="" but a # partition on this pure software mdraid that is a member # of eg md125 has FSTYPE="linux_raid_member" if dmap['FSTYPE'] == 'linux_raid_member' \ and (dnames[dname][8] is None): # N.B. 9th item (index 8) in dname = FSTYPE # We are a partition that is an mdraid raid member so # backport this info to our base device ie sda1 raid # member so label sda's FSTYPE entry the same as it's # partition's entry if the above condition is met, ie # only if the base device doesn't already have an # FSTYPE entry ie None, this way we don't overwrite # / loose info and we only need to have one partition # identified as an mdraid member to classify the entire # device (the base device) as a raid member, at least in # part. dnames[dname][8] = dmap['FSTYPE'] if ((not is_root_disk and not is_partition) or (is_btrfs)): # We have a non system disk that is not a partition # or # We have a device that is btrfs formatted # In the case of a btrfs partition we override the parted flag. # Or we may just be a non system disk without partitions. dmap['parted'] = False # could be corrected later dmap['root'] = False # until we establish otherwise as we might be. if is_btrfs: # a btrfs file system if (re.match(base_root_disk, dmap['NAME']) is not None): # We are assuming that a partition with a btrfs fs on is our # root if it's name begins with our base system disk name. # Now add the properties we stashed when looking at the base # root disk rather than the root partition we see here. dmap['SERIAL'] = root_serial dmap['MODEL'] = root_model dmap['TRAN'] = root_transport dmap['VENDOR'] = root_vendor dmap['HCTL'] = root_hctl # As we have found root to be on a partition we can now # un flag the base device as having been root prior to # finding this partition on that base_root_disk # N.B. Assumes base dev is listed before it's partitions # The 13th item in dnames entries is root so index = 12. # Only update our base_root_disk if it exists in our scanned # disks as this may be the first time we are seeing it. # Search to see if we already have an entry for the # the base_root_disk which may be us or our base dev if we # are a partition for dname in dnames.keys(): if dname == base_root_disk: dnames[base_root_disk][12] = False # And update this device as real root # Note we may be looking at the base_root_disk or one of # it's partitions there after. dmap['root'] = True # If we are an md device then use get_md_members string # to populate our MODEL since it is otherwise unused. if (re.match('md', dmap['NAME']) is not None): # cheap way to display our member drives dmap['MODEL'] = get_md_members(dmap['NAME']) else: # We have a non system disk btrfs filesystem. # Ie we are a whole disk or a partition with btrfs on but # NOT on the system disk. # Most likely a current btrfs data drive or one we could # import. # As we don't understand / support btrfs in partitions # then ignore / skip this btrfs device if it's a partition if is_partition: continue # convert size into KB size_str = dmap['SIZE'] if (size_str[-1] == 'G'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024) elif (size_str[-1] == 'T'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024 * 1024) else: # Move to next line if we don't understand the size as GB or TB # Note that this may cause an entry to be ignored if formatting # changes. # Previous to the explicit ignore swap clause this often caught # swap but if swap was in GB and above min_size then it could # show up when not in a partition (the previous caveat clause). continue if (dmap['SIZE'] < min_size): continue # No more continues so the device we have is to be passed to our db # entry system views/disk.py ie _update_disk_state() # Do final tidy of data in dmap and ready for entry in dnames dict. # db needs unique serial so provide one where there is none found. # First try harder with udev if lsblk failed on serial retrieval. if (dmap['SERIAL'] == '' or always_use_udev_serial): # lsblk fails to retrieve SERIAL from VirtIO drives and some # sdcard devices and md devices so try specialized function. dmap['SERIAL'] = get_disk_serial(dmap['NAME']) if (dmap['SERIAL'] == '' or (dmap['SERIAL'] in serials_seen)): # No serial number still or its a repeat. # Overwrite drive serial entry in dmap with fake-serial- + uuid4 # See disk/disks_table.jst for a use of this flag mechanism. # Previously we did dmap['SERIAL'] = dmap['NAME'] which is less # robust as it can itself produce duplicate serial numbers. dmap['SERIAL'] = 'fake-serial-' + str(uuid.uuid4()) # 12 chars (fake-serial-) + 36 chars (uuid4) = 48 chars serials_seen.append(dmap['SERIAL']) # replace all dmap values of '' with None. for key in dmap.keys(): if (dmap[key] == ''): dmap[key] = None # transfer our device info as now parsed in dmap to the dnames dict dnames[dmap['NAME']] = [dmap['NAME'], dmap['MODEL'], dmap['SERIAL'], dmap['SIZE'], dmap['TRAN'], dmap['VENDOR'], dmap['HCTL'], dmap['TYPE'], dmap['FSTYPE'], dmap['LABEL'], dmap['UUID'], dmap['parted'], dmap['root'], ] # Transfer our collected disk / dev entries of interest to the disks list. for d in dnames.keys(): disks.append(Disk(*dnames[d])) return disks
def scan_disks(min_size): root = root_disk() cmd = ['/usr/bin/lsblk', '-P', '-o', 'NAME,MODEL,SERIAL,SIZE,TRAN,VENDOR,HCTL,TYPE,FSTYPE,LABEL,UUID'] o, e, rc = run_command(cmd) dnames = {} disks = [] serials = [] root_serial = None # to use udevadm for serial # rather than lsblk make this True always_use_udev_serial = False for l in o: if (re.match('NAME', l) is None): continue dmap = {} cur_name = '' cur_val = '' name_iter = True val_iter = False sl = l.strip() i = 0 while i < len(sl): if (name_iter and sl[i] == '=' and sl[i + 1] == '"'): name_iter = False val_iter = True i = i + 2 elif (val_iter and sl[i] == '"' and (i == (len(sl) - 1) or sl[i + 1] == ' ')): val_iter = False name_iter = True i = i + 2 dmap[cur_name.strip()] = cur_val.strip() cur_name = '' cur_val = '' elif (name_iter): cur_name = cur_name + sl[i] i = i + 1 elif (val_iter): cur_val = cur_val + sl[i] i = i + 1 else: raise Exception('Failed to parse lsblk output: %s' % sl) if (dmap['TYPE'] == 'rom'): continue if (dmap['NAME'] == root): root_serial = dmap['SERIAL'] if (dmap['TYPE'] == 'part'): for dname in dnames.keys(): if (re.match(dname, dmap['NAME']) is not None): dnames[dname][11] = True if (((dmap['NAME'] != root and dmap['TYPE'] != 'part') or (dmap['TYPE'] == 'part' and dmap['FSTYPE'] == 'btrfs'))): dmap['parted'] = False # part = False by default dmap['root'] = False if (dmap['TYPE'] == 'part' and dmap['FSTYPE'] == 'btrfs'): # btrfs partition for root(rockstor_rockstor) pool if (re.match(root, dmap['NAME']) is not None): dmap['SERIAL'] = root_serial dmap['root'] = True else: # ignore btrfs partitions that are not for rootfs. continue # convert size into KB size_str = dmap['SIZE'] if (size_str[-1] == 'G'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024) elif (size_str[-1] == 'T'): dmap['SIZE'] = int(float(size_str[:-1]) * 1024 * 1024 * 1024) else: continue if (dmap['SIZE'] < min_size): continue if (dmap['SERIAL'] == '' or always_use_udev_serial): # lsblk fails to retrieve SERIAL from VirtIO drives and some # sdcard devices so try specialized function. # dmap['SERIAL'] = get_virtio_disk_serial(dmap['NAME']) dmap['SERIAL'] = get_disk_serial(dmap['NAME'], None) if (dmap['SERIAL'] == '' or (dmap['SERIAL'] in serials)): # No serial number still or its a repeat. # Overwrite drive serial entry with drive name: # see disks_table.jst for a use of this flag mechanism. dmap['SERIAL'] = dmap['NAME'] serials.append(dmap['SERIAL']) for k in dmap.keys(): if (dmap[k] == ''): dmap[k] = None dnames[dmap['NAME']] = [dmap['NAME'], dmap['MODEL'], dmap['SERIAL'], dmap['SIZE'], dmap['TRAN'], dmap['VENDOR'], dmap['HCTL'], dmap['TYPE'], dmap['FSTYPE'], dmap['LABEL'], dmap['UUID'], dmap['parted'], dmap['root'], ] for d in dnames.keys(): disks.append(Disk(*dnames[d])) return disks
def scan_disks(min_size): base_root_disk = root_disk() cmd = ["/usr/bin/lsblk", "-P", "-o", "NAME,MODEL,SERIAL,SIZE,TRAN,VENDOR,HCTL,TYPE,FSTYPE,LABEL,UUID"] o, e, rc = run_command(cmd) dnames = {} # Working dictionary of devices. disks = [] # List derived from the final working dictionary of devices. serials_seen = [] # List tally of serials seen during this scan. # Stash variables to pass base info on root_disk to root device proper. root_serial = root_model = root_transport = root_vendor = root_hctl = None # To use udevadm to retrieve serial number rather than lsblk, make this True # N.B. when lsblk returns no serial for a device then udev is used anyway. always_use_udev_serial = False device_names_seen = [] # List tally of devices seen during this scan for line in o: # skip processing of all lines that don't begin with "NAME" if re.match("NAME", line) is None: continue # setup our line / dev name dependant variables # easy read categorization flags, all False until found otherwise. is_base_dev = False # a base device = md126 or sda, NOT md0p2 or sda3 is_root_disk = False # the base dev that / is mounted on ie system disk is_partition = is_btrfs = False dmap = {} # dictionary to hold line info from lsblk output eg NAME: sda # line parser variables cur_name = "" cur_val = "" name_iter = True val_iter = False sl = line.strip() i = 0 while i < len(sl): # We iterate over the line to parse it's information char by char # keeping track of name or value and adding the char accordingly if name_iter and sl[i] == "=" and sl[i + 1] == '"': name_iter = False val_iter = True i = i + 2 elif val_iter and sl[i] == '"' and (i == (len(sl) - 1) or sl[i + 1] == " "): val_iter = False name_iter = True i = i + 2 dmap[cur_name.strip()] = cur_val.strip() cur_name = "" cur_val = "" elif name_iter: cur_name = cur_name + sl[i] i = i + 1 elif val_iter: cur_val = cur_val + sl[i] i = i + 1 else: raise Exception("Failed to parse lsblk output: %s" % sl) # md devices, such as mdadmin software raid and some hardware raid block # devices show up in lsblk's output multiple times with identical info. # Given we only need one copy of this info we remove duplicate device # name entries, also offers more sane output to views/disk.py where name # will be used as the index if dmap["NAME"] in device_names_seen: continue device_names_seen.append(dmap["NAME"]) # We are not interested in CD / DVD rom devices so skip to next device if dmap["TYPE"] == "rom": continue # We are not interested in swap partitions or devices so skip further # processing and move to next device. # N.B. this also facilitates a simpler mechanism of classification. if dmap["FSTYPE"] == "swap": continue # ----- Now we are done with easy exclusions we begin classification. # ------------ Start more complex classification ------------- if dmap["NAME"] == base_root_disk: # as returned by root_disk() # We are looking at the system drive that hosts, either # directly or as a partition, the / mount point. # Given lsblk doesn't return serial, model, transport, vendor, hctl # when displaying partitions we grab and stash them while we are # looking at the root drive directly, rather than the / partition. # N.B. assumption is lsblk first displays devices then partitions, # this is the observed behaviour so far. root_serial = dmap["SERIAL"] root_model = dmap["MODEL"] root_transport = dmap["TRAN"] root_vendor = dmap["VENDOR"] root_hctl = dmap["HCTL"] # Set readability flag as base_dev identified. is_root_disk = True # root as returned by root_disk() # Normal partitions are of type 'part', md partitions are of type 'md' # normal disks are of type 'disk' md devices are of type eg 'raid1'. # Disk members of eg intel bios raid md devices fstype='isw_raid_member' # Note for future re-write; when using udevadm DEVTYPE, partition and # disk works for both raid and non raid partitions and devices. # Begin readability variables assignment # - is this a partition regular or md type. if dmap["TYPE"] == "part" or dmap["TYPE"] == "md": is_partition = True # - is filesystem of type btrfs if dmap["FSTYPE"] == "btrfs": is_btrfs = True # End readability variables assignment if is_partition: # search our working dictionary of already scanned devices by name for dname in dnames.keys(): if re.match(dname, dmap["NAME"]) is not None: # Our device name has a base device entry of interest saved: # ie we have scanned and saved sdb but looking at sdb3 now. # Given we have found a partition on an existing base dev # we should update that base dev's entry in dnames to # parted "True" as when recorded lsblk type on base device # would have been disk or RAID1 (for base md device). # Change the 12th entry (0 indexed) of this device to True # The 12 entry is the parted flag so we label # our existing dnames entry as parted ie partitioned. dnames[dname][11] = True if (not is_root_disk and not is_partition) or (is_partition and is_btrfs): # We have a non system disk that is not a partition # or # We have a partition that is btrfs formatted # In the case of a btrfs partition we override the parted flag. # Or we may just be a non system disk without partitions. dmap["parted"] = False dmap["root"] = False # until we establish otherwise as we might be. if is_partition and is_btrfs: # a btrfs partition if re.match(base_root_disk, dmap["NAME"]) is not None: # We are assuming that a partition with a btrfs fs on is our # root if it's name begins with our base system disk name. # Now add the properties we stashed when looking at the base # root disk rather than the root partition we see here. dmap["SERIAL"] = root_serial dmap["root"] = True # now we have base_root_disk name match dmap["MODEL"] = root_model dmap["TRAN"] = root_transport dmap["VENDOR"] = root_vendor dmap["HCTL"] = root_hctl # and if we are an md device then use get_md_members string # to populate our MODEL since it is otherwise unused. if re.match("md", dmap["NAME"]) is not None: # cheap way to display our member drives dmap["MODEL"] = get_md_members(dmap["NAME"]) else: # ignore btrfs partitions that are not on our system disk. continue # convert size into KB size_str = dmap["SIZE"] if size_str[-1] == "G": dmap["SIZE"] = int(float(size_str[:-1]) * 1024 * 1024) elif size_str[-1] == "T": dmap["SIZE"] = int(float(size_str[:-1]) * 1024 * 1024 * 1024) else: # Move to next line if we don't understand the size as GB or TB # Note that this may cause an entry to be ignored if formatting # changes. # Previous to the explicit ignore swap clause this often caught # swap but if swap was in GB and above min_size then it could # show up when not in a partition (the previous caveat clause). continue if dmap["SIZE"] < min_size: continue # No more continues so the device we have is to be passed to our db # entry system views/disk.py ie _update_disk_state() # Do final tidy of data in dmap and ready for entry in dnames dict. # db needs unique serial so provide one where there is none found. # First try harder with udev if lsblk failed on serial retrieval. if dmap["SERIAL"] == "" or always_use_udev_serial: # lsblk fails to retrieve SERIAL from VirtIO drives and some # sdcard devices and md devices so try specialized function. dmap["SERIAL"] = get_disk_serial(dmap["NAME"]) if dmap["SERIAL"] == "" or (dmap["SERIAL"] in serials_seen): # No serial number still or its a repeat. # Overwrite drive serial entry in dmap with fake-serial- + uuid4 # See disk/disks_table.jst for a use of this flag mechanism. # Previously we did dmap['SERIAL'] = dmap['NAME'] which is less # robust as it can itself produce duplicate serial numbers. dmap["SERIAL"] = "fake-serial-" + str(uuid.uuid4()) # 12 chars (fake-serial-) + 36 chars (uuid4) = 48 chars serials_seen.append(dmap["SERIAL"]) # replace all dmap values of '' with None. for key in dmap.keys(): if dmap[key] == "": dmap[key] = None # transfer our device info as now parsed in dmap to the dnames dict dnames[dmap["NAME"]] = [ dmap["NAME"], dmap["MODEL"], dmap["SERIAL"], dmap["SIZE"], dmap["TRAN"], dmap["VENDOR"], dmap["HCTL"], dmap["TYPE"], dmap["FSTYPE"], dmap["LABEL"], dmap["UUID"], dmap["parted"], dmap["root"], ] # Transfer our collected disk / dev entries of interest to the disks list. for d in dnames.keys(): disks.append(Disk(*dnames[d])) return disks