def addPartition(self,
size,
partitionlabel,
keep=False,
fslabel=None,
fstype='',
partitionflag=[]):
"""Add a partition of given size in MiB to partition table
Where a partition for the given label exists, and is the same size, its uuid
will be returned when its filesystem is the same as fstype or if fstype is blank.
Where the partition size differs, the partition with the given partition label
will be deleted and recreated at the appropriate size.
:param: keep - when True, if the partition with given label exists keep it
:return: uuid of existing or created partition
"""
if not self.disk.confirmedAsExpectedDisk:
self.disk.confirmExpectedDisk()
existingPartition = self.getPartitionBy(partitionlabel, 'partlabel')
if not existingPartition is None:
if existingPartition.mibsize == size and keep:
if fstype == '' or fstype == existingPartition.fstype:
return (existingPartition.partuuid)
# Replacing
self.rmPartition(existingPartition)
existingPartition = None
currentDevices = set(self._listDevices())
startsector, endsector = self._findSpace(size)
bash(
f'sudo parted {self.disk.device} mkpart {partitionlabel} {fstype} {startsector}s {endsector}s'
)
self._updatePartitionTableData()
newDevice = set(self._listDevices()) - currentDevices
print(f'Making partition {partitionlabel} on {self.disk.device}')
if (len(newDevice)) == 0:
raise OSError(
f'Partition {partitionlabel} could not be created on {self.disk.device}'
)
newDevice = Path(newDevice.pop())
newPartition = self.getPartitionBy(newDevice, 'device')
for flag in partitionflag:
bash(
f'sudo parted {self.disk.device} set {newPartition.partnbr} {flag} on'
)
print('Partition table written. Checking alignment')
print(
bash(
f'sudo parted {self.disk.device} align-check optimal {newPartition.partnbr}'
).stdout)
newPartition.mkfs(fstype, fslabel)
return (newPartition.partuuid)
def _updatePartitionTableData(self):
"""Use sfdisk, parted and blkid to read partition table information from disk"""
if self.disk is None:
raise ValueError("Disk must be set to read partition table")
self._data['sfdisk']=\
ast.literal_eval(bash(f'sudo sfdisk -l -J {self.disk.device}').stdout)
self._data['parted']=\
list(map(
lambda x:x.rstrip(';').split(':'),
filter(
lambda x:re.match('[0-9]',x) is not None,
bash(f'sudo parted -m {self.disk.device} unit s print')
.stdout
.splitlines()
)
))
self._data['blkid']=getDictionaryFromKeyValueString(\
bash(f'sudo blkid -o export {self.disk.device}').stdout)
self.firstlba = self._data['sfdisk']['partitiontable']['firstlba']
self.lastlba = self._data['sfdisk']['partitiontable']['lastlba']
self.bytessector = self._data['sfdisk']['partitiontable']['sectorsize']
def _collatePartitionData(sfdiskData):
device = Path(sfdiskData['node'])
blkidData=getDictionaryFromKeyValueString(\
bash(f'sudo blkid -o export {device}').stdout)
start = int(sfdiskData['start'])
partedData = list(
filter(lambda x: int(x[1].rstrip('s')) == start,
self._data['parted'])).pop()
data = {
'device': device,
'startsector': start,
'endsector': int(partedData[2].rstrip('s')),
'sectorcount': int(sfdiskData['size']),
'fstype': partedData[4],
'fslabel': blkidData['LABEL'],
'fsuuid': blkidData['UUID'],
'partlabel': sfdiskData['name'],
'partuuid': sfdiskData['uuid'],
'partflag': partedData[6],
'partnbr': int(partedData[0])
}
self.partitiondata[device] = data #By device
self.partitiondata[sfdiskData['name']] = data
self.partitiondata[sfdiskData['uuid']] = data #By uuid
self.partitiondata[blkidData['UUID']] = data #By fsuuid
if 'partitions' in self._data['sfdisk']['partitiontable']:
[
_collatePartitionData(d)
for d in self._data['sfdisk']['partitiontable']['partitions']
]
self._updatePartitions()
def lvm_backup(vg: str, lv: str, out_p: Path, verbose: bool = True) -> Path:
"""Creates a snapshot of a logical volume, mounts it in temporary directory
creates an tar gzip archive in out_path and cleans up the snapshot afterwards.
Returns a path to final archive containing backed up files."""
opts = ExecOpts(quit=True, collect=False)
system.install_pkg_if_bin_not_exists("tar")
system.install_pkg_if_bin_not_exists("pigz")
try:
snapshot = lvm_snapshot(vg, lv, opts=opts)
out_p = out_p / (snapshot + ".tgz")
with TemporaryDirectory() as tempdir:
inp_p = Path(tempdir) / snapshot
inp_p.mkdir(parents=True)
try:
Command("mount", [f"/dev/{vg}/{snapshot}",
str(inp_p)],
opts=opts).safe_run()
flags = "-cvf" if verbose else "-cf"
bash(
f"cd {str(inp_p)} && tar -I pigz {flags} {str(out_p)} ./*",
quit=True,
)
finally:
Command("umount", [f"/dev/{vg}/{snapshot}"],
opts=opts).safe_run()
finally:
if "snapshot" in locals():
lvm_remove(vg, snapshot)
return out_p
def confirmExpectedDisk(self):
print('Is this disk the right disk?')
print(bash(f'sudo sfdisk -l {self.device}').stdout)
userConfirm('Is this the right disk? Y|N')
print('Disk has the following partitions, confirm each one')
print(bash(f'sudo parted -l {self.device}').stdout)
#userConfirm('Are these the right partitions? Y|N')
print('Disk now cleared to be have its partition table modified.')
self.confirmedAsExpectedDisk = True
def mkfs(self, typevalue, label):
"""Format partition filesystem and run pending callbacks
:note: Only fs types in Disk.Partition.MKFS_FSTYPE will be
prepared
"""
if typevalue in Partition.MKFS_FSTYPE:
self.fslabel = label
print(f'Building {typevalue} filesystem on {self.device}')
bash(f'sudo mkfs -t {typevalue} -L {self.fslabel} {self.device}')
for f in self.doonmkfs:
f()
def __init__(self, devicepath):
self.device = devicepath
self.partitiontable = PartitionTable(self)
self.alignmentoffset = Disk.ALIGNMENT_OFFSET
self.confirmedAsExpectedDisk = False
self.bytecount = int(
bash(f'sudo blockdev --getsize64 {self.device}').stdout)
self.GiBcount = int(self.bytecount / (pow(2, 30)))
self.bytessector = int(
bash(f'sudo blockdev --getpbsz {self.device}').stdout)
self.sectorcount = self.bytecount / self.bytessector
def build_paru():
install_sudo()
bld_pkgs = ["base-devel"]
if not bins_exist(bld_pkgs):
install_pkgs(bld_pkgs)
with TemporaryDirectory() as tmpdir:
gitclone(PARU_REPO, Path(f"{tmpdir}/paru"))
chown(tmpdir, "nobody", "nobody")
bash(f"usermod -d {tmpdir} nobody")
sudo_nopasswd("nobody")
bash(f"cd {tmpdir}/paru && sudo -u nobody makepkg -srci --noconfirm")
rm_sudo_nopasswd("nobody")
def _collatePartitionData(sfdiskData):
device = Path(sfdiskData['node'])
blkidData=getDictionaryFromKeyValueString(\
bash(f'sudo blkid -o export {device}').stdout)
start = int(sfdiskData['start'])
partedData = list(
filter(lambda x: int(x[1].rstrip('s')) == start,
self._data['parted'])).pop()
data = {
'device': device,
'startsector': start,
'endsector': int(partedData[2].rstrip('s')),
'sectorcount': int(sfdiskData['size']),
'fstype': partedData[4],
'fslabel': blkidData['LABEL'],
'fsuuid': blkidData['UUID'],
'partlabel': sfdiskData['name'],
'partuuid': sfdiskData['uuid'],
'partflag': partedData[6],
'partnbr': int(partedData[0])
}
self.partitiondata[device] = data #By device
self.partitiondata[sfdiskData['name']] = data
self.partitiondata[sfdiskData['uuid']] = data #By uuid
self.partitiondata[blkidData['UUID']] = data #By fsuuid
def get_version(self):
if self.is_available():
version_str = bash(["clang-format", "--version"], log=False)
m = re.search(
r"version (?P<version>\d+\.\d+\.\d+) \([\w\/:.-]+ (?P<hash>[a-z0-9]+)\)$",
version_str.strip())
if m:
version = (*(int(i) for i in m.group('version').split('.')),
m.group('hash'))
return version
def __init__(self, partition):
"""
:sideeffect: Creates tempdir and mounts parititon there if not
already mounted. This will be cleaned up on object.__del__()
"""
#Check false for if grep finds nothing
mountsline = bash(
f'cat {Partition.Mountpoint.SYSTEM_MOUNTS} | grep {partition.device}',
check=False)
if mountsline.returncode == 0:
mountinfo = mountsline.stdout.split()
self.mountpoint = Path(mountinfo[1])
self.mountoptions = mountinfo[3]
else:
self.tempmount = Path(tempfile.mkdtemp())
self.mountpoint = self.tempmount
bash(f'sudo mount {partition.device} {self.mountpoint}')
def _installGrub(self):
'''Installs grub to the grub data partition once its fs has been intialized
This is invoked once a filesystem for this partition exists.
'''
# Install grub, specifying path to grub data partition
# NB: Grub will make this path relative to the filesystems root,
# see function grub_make_system_path_relative_to_its_root_os of grub source
option=f'--boot-directory={self.grubpartition.getMountpoint()}'
print(f"Installing grub to disk {self.device}'s mbr, boot partition {self.grubbootpartition} & grub data partition {self.grubpartition}")
result = bash(f'sudo grub-install {option} {self.device}')
print(result.stdout)
def updateGrubMenu(self):
'''Updates the grub menu in grub_data'''
# Write menu file. Disable all others.
resetExecuteFile=[]
for entry in os.scandir(Path(GRUB_SYSTEM_CONFIG_DIR)):
if entry.is_file() and not entry.name in GRUB_MENU_WHITELIST:
entryStat=entry.stat()
if stat.filemode(entryStat.st_mode)[-1]=='x':
resetExecuteFile.append(entry.path)
bash(f'sudo chmod -x {entry.path}')
menuFilePath=Path(ROOT,GRUB_MENU_FILE)
menuFile=open(menuFilePath,'wt')
menuFile.write(EMPTY_GRUB_CUSTOM_MENU)
[menuFile.write(entry) for entry in self.grubmenu ]
menuFile.close()
bash(f'sudo chmod +x {menuFilePath}')
bash(f'sudo cp {menuFilePath} {GRUB_SYSTEM_CONFIG_DIR}')
bash(f'sudo grub-mkconfig -o {self.grubpartition.mountpoint}')
os.unlink(menuFilePath)
[bash(f'sudo chmod +x {p}') for p in resetExecuteFile]
def cleanup(workingDirectory, mounts):
for m in mounts:
bash(f'sudo umount {m}')
shutil.rmtree(workingDirectory)
def __del__(self):
'''Unmounts and deletes any temporary mountpoint that exists for ptn'''
if hasattr(self, 'mountpoint'):
bash(f'sudo umount {self.mountpoint}')
bash(f'rmdir {self.mountpoint}')
def gen_fstab(self):
bash(
f"/usr/bin/genfstab -U {self.location} >> {self.location}/etc/fstab",
quit=True,
)
def rmPartition(self, partition: 'Partition'):
bash(f'sudo parted {self.disk.device} rm {partition.partnbr}')
self._updatePartitionTableData()
def getFirstPartitionOffset(imageFile):
data = json.loads(bash('sfdisk -l {} -J'.format(imageFile)).stdout)
sectorsize = data['partitiontable']['sectorsize']
firstsector = min(
[n['start'] for n in data['partitiontable']['partitions']])
return (sectorsize * firstsector)
def archive_scripts(self):
bash(
f"cd {FULLPATH.parent} && cp ../*.py . && zip -r {self.location}/{FILENAME} ./*.py",
quit=True,
)
def install_vim_plug(self):
f = self.xdg_conf_dir().joinpath("nvim/autoload/plug.vim")
url = "https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim"
if not f.exists():
bash(f"curl -fLo {f} --create-dirs {url}")
system.chown(self.xdg_conf_dir(), self.username, self.username)
def run_tests(build_dir: PosixPath, test_filter_str=""):
command = ["./test.sh", "popart", "-j16"]
if test_filter_str:
command += ["-R", test_filter_str]
bash(command, build_dir, ignore_return_code=True)
osdata = config['os'][osid]
ostarball = Path(osdata['tarballpath'])
zipImageFile = ostarball.parts[-1]
imageFile = ostarball.stem
# Install openelec, grub menu entry
if not ostarball.exists():
print(f'Retrieving {osid} image')
dlFile(osdata['tarballurl'], ostarball)
shutil.copy(ostarball, workingDirectory)
print(f'Extracting {osid} image')
bash(f'gunzip {zipImageFile}')
print(f'Mounting {osid} image')
offset = getFirstPartitionOffset(imageFile)
mountpoint = Path(workingDirectory, imageFile + '.mnt')
mounts.append(mountpoint)
bash(f'mkdir {mountpoint}')
bash(f'sudo mount -o loop,offset={offset} {imageFile} {mountpoint}')
systemPartition = disk.partitiontable.getPartitionBy(
osdata['partition']['system']['partitionlabel'], 'partlabel')
print(
f'Transferring files in mounted {osid} image to {systemPartition}')
# Viashell for explansion of glob
bash(f'sudo cp -r {mountpoint}/* {systemPartition.getMountpoint()}',
