def bucket_exists(bucket_name):
"""
Verify if bucket_name exits.
Parameters
----------
bucket_name: str
The bucket_name.
Returns
-------
dict: The bucket on success, raise an exception otherwise
"""
cmd = [
'oci', 'os', 'object', 'list', '--all', '--bucket-name', bucket_name
]
_logger.debug('__ Running %s.', cmd)
pause_msg(cmd)
try:
bucket_result = json.loads(
system_tools.run_popen_cmd(cmd)['output'].decode('utf-8'))
_logger.debug('Result: \n%s', bucket_result)
return bucket_result
except Exception as e:
_logger.debug(
'Bucket %s does not exists or the authorisation is missing: %s.',
bucket_name, str(e))
raise OciMigrateException(
'Bucket %s does not exists or the authorisation is missing:' %
bucket_name) from e
def exec_vgscan(vgscan_args):
"""
Scan the system for (new) volume groups.
Parameters
----------
vgscan_args: list
list of strings, arguments for vgscan
Returns
-------
bool: True on success, raises an exeception on failure.
"""
cmd = ['vgscan'] + vgscan_args
_logger.debug('__ Executing %s' % cmd)
pause_msg(cmd)
try:
output = run_popen_cmd(cmd).decode('utf-8')
_logger.debug('Volume groups scanned:\n%s' % str(output))
return True
except Exception as e:
#
# vgscan failed
_logger.critical(' Failed to scan for volume groups: %s' % str(e))
raise OciMigrateException('Failed to scan for volume groups: %s' %
str(e))
def mount_partition(devname, mountpoint=None):
"""
Create the mountpoint /mnt/<last part of device specification> and mount a
partition on on this mountpoint.
Parameters
----------
devname: str
The full path of the device.
mountpoint: str
The mountpoint, will be generated if not provided.
Returns
-------
str: The mounted partition on Success, None otherwise.
"""
_logger.debug('__ Mount partition %s.', devname)
#
# create mountpoint /mnt/<devname> if not specified.
if mountpoint is None:
mntpoint = migrate_data.loopback_root + '/' + devname.rsplit('/')[-1]
_logger.debug('Loopback mountpoint: %s', mntpoint)
try:
if system_tools.exec_mkdir(mntpoint):
_logger.debug('Mountpoint: %s created.', mntpoint)
except Exception as e:
_logger.critical(' Failed to create mountpoint %s: %s', mntpoint, str(e))
raise OciMigrateException('Failed to create mountpoint %s:' % mntpoint) from e
else:
mntpoint = mountpoint
#
# actual mount
cmd = ['mount', devname, mntpoint]
pause_msg(cmd, pause_flag='_OCI_MOUNT')
_, nbcols = terminal_dimension()
try:
mountpart = ProgressBar(nbcols, 0.2,
progress_chars=['mount %s' % devname])
mountpart.start()
_logger.debug('command: %s', cmd)
cmdret = system_tools.run_call_cmd(cmd)
if cmdret == 0:
_logger.debug('%s mounted on %s.', devname, mntpoint)
return mntpoint
raise Exception('Mount %s failed: %d' % (devname, cmdret))
except Exception as e:
#
# mount failed, need to remove mountpoint.
_logger.critical(' Failed to mount %s, missing driver, filesystem corruption...: %s', devname, str(e))
if mountpoint is None:
if system_tools.exec_rmdir(mntpoint):
_logger.debug('%s removed', mntpoint)
else:
_logger.critical(' Failed to remove mountpoint %s', mntpoint)
finally:
if system_tools.is_thread_running(mountpart):
mountpart.stop()
return None
def mount_pseudo(rootdir):
"""
Remount proc, sys and dev.
Parameters
----------
rootdir: str
The mountpoint of the root partition.
Returns
-------
list: The list of new mountpoints on success, None otherwise.
"""
pseudodict = {'proc': ['-t', 'proc', 'none', '%s/proc' % rootdir],
'dev': ['-o', 'bind', '/dev', '%s/dev' % rootdir],
'sys': ['-o', 'bind', '/sys', '%s/sys' % rootdir]}
pseudomounts = []
_logger.debug('__ Mounting: %s', pseudodict)
for dirs, cmd_par in list(pseudodict.items()):
cmd = ['mount'] + cmd_par
_logger.debug('Mounting %s', dirs)
pause_msg(cmd, pause_flag='_OCI_MOUNT')
try:
_logger.debug('Command: %s', cmd)
cmdret = run_call_cmd(cmd)
_logger.debug('%s : %d', cmd, cmdret)
if cmdret != 0:
_logger.error(' Failed to %s', cmd)
raise Exception('%s Failed: %d' % (cmd, cmdret))
pseudomounts.append(cmd_par[3])
except Exception as e:
_logger.critical(' Failed to %s: %s', cmd, str(e))
raise OciMigrateException('Failed to %s:' % cmd) from e
return pseudomounts
def exec_lvscan(lvscan_args):
"""
Scan the system for logical volumes.
Parameters
----------
lvscan_args: list
list of strings, arguments for lvscan
Returns
-------
list: A list of strings, the output of lvscan --verbose on success,
raises an exeception on failure.
"""
cmd = ['lvscan'] + lvscan_args
_logger.debug('__ Running: %s' % cmd)
pause_msg(cmd)
try:
_logger.debug('command: %s' % cmd)
output = run_popen_cmd(cmd).decode('utf-8')
_logger.debug('Logical volumes scanned:\n%s' % str(output))
return output.splitlines()
except Exception as e:
#
# lvscan failed
_logger.critical(' Failed to scan for logical volumes: %s' % str(e))
raise OciMigrateException('Failed to scan for logical volume: %s' %
str(e))
def upload_image(imgname, bucket_name, ociname):
"""
Upload the validated and updated image imgname to the OCI object storage
bucket_name as ociname.
Parameters
----------
imgname: str
The on-premise custom image.
bucket_name: str
The OCI object storage name.
ociname:
The OCI image name.
Returns
-------
bool: True on success, raises an exception otherwise.
"""
cmd = ['oci', 'os', 'object', 'put',
'--bucket-name', bucket_name,
'--file', imgname,
'--name', ociname,
'--part-size', '100',
'--parallel-upload-count', '6']
_logger.debug('__ Running %s', cmd)
pause_msg(cmd)
try:
upload_result = system_tools.run_popen_cmd(cmd)['output'].decode('utf-8')
_logger.debug('Successfully uploaded %s to %s as %s: %s.', imgname, bucket_name, ociname, upload_result)
return True
except Exception as e:
_logger.critical(' Failed to upload %s to object storage %s as %s: %s.',
imgname, bucket_name, ociname, str(e))
raise OciMigrateException('Failed to upload %s to object storage %s as %s:'
% (imgname, bucket_name, ociname)) from e
def mount_fs(mountpoint):
"""
Mount a filesystem specified in fstab, by mountpoint only.
Parameters
----------
mountpoint: str
The mountpoint.
Returns
-------
bool: True on success, False otherwise
"""
cmd = ['mount', mountpoint]
pause_msg(cmd)
_logger.debug('__ Mounting %s' % mountpoint)
try:
_, nbcols = terminal_dimension()
mountwait = ProgressBar(nbcols,
0.2,
progress_chars=['mounting %s' % mountpoint])
mountwait.start()
_logger.debug('Command: %s' % cmd)
cmdret = run_call_cmd(cmd)
_logger.debug('%s returned %d' % (cmd, cmdret))
if cmdret == 0:
return True
else:
raise Exception('%s failed: %d' % (cmd, cmdret))
except Exception as e:
_logger.error(' Failed to %s: %s' % (cmd, str(e)))
return False
finally:
if is_thread_running(mountwait):
mountwait.stop()
def unmount_something(mountpoint):
"""
Unmount.
Parameters
----------
mountpoint: str
The mountpoint.
Returns
-------
bool: True on success, False otherwise.
"""
_logger.debug('__ Unmount %s.' % mountpoint)
if os.path.ismount(mountpoint):
_logger.debug('%s is a mountpoint.' % mountpoint)
else:
_logger.debug('%s is not a mountpoint, quitting' % mountpoint)
return True
#
cmd = ['umount', mountpoint]
pause_msg(cmd)
try:
_logger.debug('command: %s' % cmd)
cmdret = run_call_cmd(cmd)
_logger.debug('%s : %d' % (cmd, cmdret))
if cmdret != 0:
raise Exception('%s failed: %d' % (cmd, cmdret))
except Exception as e:
_logger.error(' Failed to %s: %s' % (cmd, str(e)))
return False
return True
def b20_install_cloud_agent(self):
"""
Install the oracle cloud agent.
Returns
-------
bool: True on success, False otherwise. (always True as failing to
install the oracle cloud agent is not fatal.
"""
_logger.debug('__ Install oracle cloud agent.')
if bool(migrate_data.oracle_cloud_agent_location):
_logger.debug('oracle cloud agent present: %s',
migrate_data.oracle_cloud_agent_location)
else:
_logger.debug('No oracle cloud agent package present, skipping.')
return True
#
# get package manipulation tool.
pkg_mgr = self._exec_yum \
if self.package_tool['pkg_mgr'] == 'yum' else self._exec_dnf
#
# install rpm
oracle_cloud_agent_rpm = migrate_data.oracle_cloud_agent_location
simple_rpm = oracle_cloud_agent_rpm.split('/')[-1]
try:
install_output = pkg_mgr(
self.package_tool['package_localinstall'] +
[oracle_cloud_agent_rpm])
_logger.debug('Successfully installed pkg %s:\n%s', simple_rpm,
install_output)
migrate_tools.result_msg(msg='Installed %s.' % simple_rpm,
result=False)
pause_msg('cloud agent', pause_flag='_OCI_AGENT')
except Exception as e:
_logger.warning('Failed to install %s: %s', simple_rpm, str(e))
return True
def exec_vgs_noheadings():
"""
List the local volume group and generates a new (temporary) name as
a hex UUID.
Returns
-------
list: list of lists [orginal volume group name, new volume group name].
"""
cmd = ['vgs', '--noheadings']
_logger.debug('__ Executing %s.' % cmd)
pause_msg(cmd)
vg_list = list()
try:
vgs_response = run_popen_cmd(cmd)
output = vgs_response.decode('utf-8').splitlines() if bool(
vgs_response) else b''
if bool(output):
for vg_record in output:
if len(vg_record) > 0:
vg_list.append([vg_record.split()[0], uuid.uuid4().hex])
_logger.debug('Volume groups found: %s' % vg_list)
return vg_list
except Exception as e:
_logger.critical(' Failed to list current volume groups: %s' %
str(e))
def exec_pvscan(pvscan_args, devname=None):
"""
Update the lvm cache.
Parameters
----------
pvscan_args: list
List of strings, arguments for pvscan
devname: str
Device name to scan.
Returns
-------
bool: True on success, raises an exception on failure.
"""
_logger.debug('__ Running pvscan %s' % pvscan_args)
cmd = ['pvscan'] + pvscan_args
if devname is not None:
cmd.append(devname)
pause_msg(cmd)
try:
_logger.debug('command: %s' % cmd)
cmdret = run_call_cmd(cmd)
_logger.debug('Physical volumes scanned on %s: %d' % (devname, cmdret))
if cmdret != 0:
_logger.error(' Physical volume scan failed.')
raise Exception('Physical volume scan failed.')
return True
except Exception as e:
#
# pvscan failed
_logger.critical(' Failed to scan %s for physical volumes: %s' %
(devname, str(e)))
raise OciMigrateException('Failed to scan %s for physical '
'volumes: %s' % (devname, str(e)))
def get_cloud_agent_if_relevant(root_part, os_type, major_release):
"""
Download oracle cloud agent for os where this cannot be achieved via
standard install channels.
Parameters
----------
root_part: str
The root partition of the image as currently mounted.
os_type: str
The os type id as set in the os-release file.
major_release: str
The major release info.
Returns
-------
bool: always True
"""
_logger.debug('__ Collecting the oracle_cloud_agent package if relevant.')
_logger.debug('root partition: %s' % root_part)
_logger.debug('os type: %s' % os_type)
_logger.debug('major release: %s' % major_release)
pause_msg('cloud agent', pause_flag='_OCI_AGENT')
try:
if os_type in get_config_data('ol_os_for_cloud_agent'):
_logger.debug('OS is OL-type os.')
el_tag = get_config_data('ol_version_id_dict')[major_release]
_logger.debug('el tag: %s' % el_tag)
cloud_agent_url = get_url_data_from_base(el_tag)
_logger.debug('cloud agent url: %s' % cloud_agent_url)
if cloud_agent_url is None:
_logger.debug('URL with oracle cloud agent package not found.')
else:
package_name = os.path.basename(cloud_agent_url)
destination_dir = get_config_data(
'ol_os_oracle_cloud_agent_store')
destination = os.path.join(root_part,
destination_dir.lstrip('/'),
package_name.lstrip('/'))
_logger.debug(
'Destination for oracle cloud agent package: %s' %
destination)
_ = get_file_from_url(cloud_agent_url, destination)
migrate_data.oracle_cloud_agent_location = \
os.path.join('/',
destination_dir.lstrip('/'),
package_name.lstrip('/'))
_logger.debug('cloud agent location: %s' %
migrate_data.oracle_cloud_agent_location)
else:
_logger.debug('This operation is not relevant here.')
return True
except Exception as e:
_logger.debug('Installation of the oracle cloud agent failed: %s' %
str(e))
pause_msg('cloud agent', pause_flag='_OCI_AGENT')
return True
def a10_remove_cloud_init(self):
"""
Remove cloud-init package and configuration
Returns
-------
bool: True on success, False otherwise.
"""
_logger.debug('__ Remove existing cloud_init config.')
try:
#
# write 90_dpkg.cfg
dpkg_cfg_path = '/etc/cloud/cloud.cfg.d'
if os.path.exists(dpkg_cfg_path):
with open(dpkg_cfg_path + '/90_dpkg.cfg', 'w') as f:
f.write('datasource_list: [None]\n')
_logger.debug('%s (re)written.', dpkg_cfg_path)
else:
_logger.debug('%s does not exist.', dpkg_cfg_path)
#
# remove cloud-init package
purge_output = self._exec_apt(['purge', 'cloud-init', '-y'])
_logger.debug('cloud-init purged: %s', purge_output)
#
# remove /etc/cloud
cloud_cfg_path = '/etc/cloud'
backup_cloud_path = system_tools.exec_rename(cloud_cfg_path)
if bool(backup_cloud_path):
_logger.debug('%s renamed to %s', cloud_cfg_path,
backup_cloud_path)
#
# remove /var/lib/cloud
var_lib_cloud_path = '/var/lib/cloud'
backup_var_lib_cloud_path = system_tools.exec_rename(
var_lib_cloud_path)
if bool(backup_var_lib_cloud_path):
_logger.debug('%s renamed to %s', var_lib_cloud_path,
backup_var_lib_cloud_path)
#
# remove logs
cloud_init_log = '/var/log/cloud-init.log'
backup_cloud_init_log = system_tools.exec_rename(cloud_init_log)
if bool(cloud_init_log):
_logger.debug('%s renamed to %s', cloud_init_log,
backup_cloud_init_log)
#
pause_msg(msg='cloud-init removed', pause_flag='_OCI_CHROOT')
return True
except Exception as e:
_logger.warning(
'Failed to purge cloud-init completely which might cause issues '
'at instance creation: %s', str(e))
return False
def exec_parted(devname):
"""
Collect data about the device on the image using the parted utility.
Parameters
----------
devname: str
The device name.
Returns
-------
dict: The device data from parted utility on success, None otherwise.
"""
cmd = ['parted', devname, 'print']
_logger.debug('__ Running %s' % cmd)
pause_msg(cmd)
try:
result = run_popen_cmd(cmd).decode('utf-8')
_logger.debug('parted: %s' % result)
device_data = dict()
device_data['Partition List'] = list()
for devx in result.splitlines():
_logger.debug('%s - %d' % (devx, len(devx)))
if 'Model' in devx:
device_data['Model'] = devx.split(':')[1]
_logger.debug('Model %s' % device_data['Model'])
elif 'Disk Flags' in devx:
device_data['Disk Flags'] = devx.split(':')[1]
_logger.debug('Disk Flags %s' % device_data['Disk Flags'])
elif 'Disk' in devx:
device_data['Disk'] = devx.split(':')[1]
_logger.debug('Disk %s' % device_data['Disk'])
elif 'Partition Table' in devx:
device_data['Partition Table'] = devx.split(':')[1]
_logger.debug('Partition Table %s' %
device_data['Partition Table'])
elif devx.split():
if devx.split()[0].isdigit():
device_data['Partition List'].append(devx.split())
_logger.debug('Partition: %s' % devx.split())
else:
_logger.debug('Ignoring %s' % devx)
else:
_logger.debug('Ignoring %s' % devx)
_logger.debug(device_data)
pause_msg(device_data)
return device_data
except Exception as e:
_logger.error(' Failed to collect parted %s device data: %s' %
(devname, str(e)))
return None
def mount_imgfn(imgname):
"""
Link vm image with an nbd device.
Parameters
----------
imgname: str
Full path of the image file.
Returns
-------
str: Device name on success, raises an exception otherwise.
"""
#
# create nbd devices
_logger.debug('__ Running mount image file %s.' % imgname)
result_msg(msg='Load nbd')
if not system_tools.create_nbd():
raise OciMigrateException('Failed ot load nbd module')
else:
_logger.debug('nbd module loaded')
#
# find free nbd device
result_msg(msg='Find free nbd device')
devpath = system_tools.get_free_nbd()
_logger.debug('Device %s is free.' % devpath)
#
# link img with first free nbd device
result_msg(msg='Mount image %s' % imgname, result=True)
_, nbcols = terminal_dimension()
try:
mountwait = ProgressBar(nbcols, 0.2, progress_chars=['mounting image'])
mountwait.start()
qemucmd = ['-c', devpath, imgname]
pause_msg(qemucmd)
qemunbd_ret = system_tools.exec_qemunbd(qemucmd)
if qemunbd_ret == 0:
time.sleep(5)
_logger.debug('qemu-nbd %s succeeded' % qemucmd)
return devpath
else:
_logger.critical('\n Failed to create nbd devices: %d' %
qemunbd_ret)
raise Exception('Failed to create nbd devices: %d' % qemunbd_ret)
except Exception as e:
_logger.critical('\n Something wrong with creating nbd devices: %s' %
str(e))
raise OciMigrateException('Unable to create nbd devices: %s' % str(e))
finally:
if system_tools.is_thread_running(mountwait):
mountwait.stop()
def unmount_part(devname):
"""
Unmount a partition from mountpoint from /mnt/<last part of device
specification> and remove the mountpoint.
Parameters
----------
devname: str
The full path of the device.
Returns
-------
bool
True on success, False otherwise.
"""
mntpoint = migrate_data.loopback_root + '/' + devname.rsplit('/')[-1]
cmd = ['umount', mntpoint]
_logger.debug('__ Running %s' % cmd)
pause_msg(cmd)
while True:
try:
_logger.debug('command: %s' % cmd)
cmdret = system_tools.run_call_cmd(cmd)
if cmdret == 0:
_logger.debug('%s unmounted from %s' % (devname, mntpoint))
#
# remove mountpoint
if system_tools.exec_rmdir(mntpoint):
_logger.debug('%s removed' % mntpoint)
return True
else:
_logger.critical(' Failed to remove mountpoint %s' %
mntpoint)
raise OciMigrateException(
'Failed to remove mountpoint %s' % mntpoint)
else:
_logger.critical(' Failed to unmount %s: %d' %
(devname, cmdret))
console_msg('Failed to unmount %s, error code %d.\n '
'Please verify before continuing.' %
(devname, cmdret))
retry = read_yn(
'Something prevented to complete %s, please '
'verify and correct if possible. '
'Press Y to retry, N to ignore.',
waitenter=True)
if not retry:
break
except Exception as e:
_logger.critical(' Failed to unmount %s: %s' % (devname, str(e)))
return False
def exec_sfdisk(devname):
"""
Collect the data about the partitions on the image file mounted on the
device devname using the sfdisk utility.
Parameters
----------
devname: str
The device.
Returns
-------
dict: The partition data with sfdisk results on success, None otherwise.
"""
cmd = ['sfdisk', '-d', devname]
_logger.debug('__ Running %s' % cmd)
pause_msg(cmd)
try:
result = run_popen_cmd(cmd).decode('utf-8')
partdata = dict()
for devx in result.splitlines():
if devx.startswith(devname):
key = devx.split(':')[0].strip()
migrate_tools.result_msg(msg='sfdisk partition %s' % key,
result=False)
thispart = {'start': 0, 'size': 0, 'Id': 0, 'bootable': False}
val = devx.split(':')[1].split(',')
for it in val:
if 'start' in it:
x = it.split('=')[1]
thispart['start'] = int(x)
elif 'size' in it:
x = it.split('=')[1]
thispart['size'] = int(x)
elif 'Id' in it:
x = it.split('=')[1]
thispart['Id'] = x.strip()
elif 'bootable' in it:
thispart['bootable'] = True
else:
_logger.debug('unrecognised item: %s' % val)
partdata[key] = thispart
_logger.debug(partdata)
return partdata
except Exception as e:
_logger.error(' Failed to collect sfdisk %s partition data: %s' %
(devname, str(e)))
return None
def exec_ldconfig():
"""
Executes ldconfig to update the shared library cache.
Returns
-------
int: 0 on success, raises an exception otherwise.
"""
cmd = ['ldconfig']
_logger.debug('__ Running %s' % cmd)
try:
pause_msg('running ldconfig')
return run_call_cmd(cmd)
except Exception as e:
_logger.error(' %s command failed: %s' % (cmd, str(e)))
raise OciMigrateException('\n%s command failed: %s' % (cmd, str(e)))
def a10_remove_cloud_init(self):
"""
Remove the cloud-init software and the configuration data.
Returns
-------
bool: True on success, False otherwise.
"""
_logger.debug('__ Remove existing cloud_init config.')
try:
pkg_mgr = self._exec_yum \
if self.package_tool['pkg_mgr'] == 'yum' else self._exec_dnf
package_erase = self.package_tool['package_remove'] + [
'cloud-init'
]
remove_output = pkg_mgr(package_erase)
_logger.debug('cloud-init removed: %s', remove_output)
#
# remove /etc/cloud
cloud_cfg_path = '/etc/cloud'
backup_cloud_path = system_tools.exec_rename(cloud_cfg_path)
if bool(backup_cloud_path):
_logger.debug('%s renamed to %s', cloud_cfg_path,
backup_cloud_path)
#
# remove /var/lib/cloud
var_lib_cloud_path = '/var/lib/cloud'
backup_var_lib_cloud_path = system_tools.exec_rename(
var_lib_cloud_path)
if bool(backup_var_lib_cloud_path):
_logger.debug('%s renamed to %s', var_lib_cloud_path,
backup_var_lib_cloud_path)
#
# remove logs
cloud_init_log = '/var/log/cloud-init.log'
backup_cloud_init_log = system_tools.exec_rename(cloud_init_log)
if bool(cloud_init_log):
_logger.debug('%s renamed to %s', cloud_init_log,
backup_cloud_init_log)
#
pause_msg(msg='cloud-init removed', pause_flag='_OCI_CHROOT')
return True
except Exception as e:
_logger.warning(
'Failed to remove cloud-init completely which might cause issues at instance '
'creation: %s', str(e))
return False
def unmount_imgfn(devname):
"""
Unlink a device.
Parameters
----------
devname: str
The device name
Returns
-------
bool: True on succes, raise an exception otherwise.
"""
_logger.debug('__ Unmount %s' % devname)
try:
#
# release device
qemucmd = ['-d', devname]
pause_msg(qemucmd)
qemunbd_ret = system_tools.exec_qemunbd(qemucmd)
if qemunbd_ret == 0:
_logger.debug('qemu-nbd %s succeeded: %d' % (qemucmd, qemunbd_ret))
else:
raise Exception('%s returned %d' % (qemucmd, qemunbd_ret))
#
# clear lvm cache, if necessary.
if system_tools.exec_pvscan(['--cache']):
_logger.debug('lvm cache updated')
else:
_logger.error(' Failed to clear LVM cache.')
raise OciMigrateException('Failed to clear LVM cache.')
#
# remove nbd module
if not system_tools.rm_nbd():
raise OciMigrateException('Failed to remove nbd module.')
else:
_logger.debug('Successfully removed nbd module.')
except Exception as e:
_logger.critical(' Something wrong with removing nbd devices: %s' %
str(e))
raise OciMigrateException('\nSomething wrong with removing nbd '
'devices: %s' % str(e))
return True
def exec_rename_volume_groups(vg_list, direction):
"""
Rename a list of volume groups.
Parameters
----------
vg_list: list
list of lists [original name, new name]
direction: str
if FORWARD, rename original name to new name, if BACKWARD from new name
to original name.
Returns
-------
bool: True on success, False otherwise.
"""
_logger.debug('__ Rename volume group %s.' % vg_list)
result = True
#
for vg_names in vg_list:
if direction == 'FORWARD':
cmd = ['vgrename', vg_names[0], vg_names[1]]
elif direction == 'BACKWARD':
cmd = ['vgrename', vg_names[1], vg_names[0]]
else:
_logger.debug('Invalid argument %s' % direction)
return False
#
pause_msg(cmd)
try:
_logger.debug('command: %s' % cmd)
output = run_popen_cmd(cmd).decode('utf-8')
if 'successfully renamed' in output:
_logger.debug('%s succeeded' % cmd)
else:
_logger.debug('%s failed' % cmd)
result = False
except Exception as e:
_logger.debug('Execution of vgrename failed: %s' % str(e))
result = False
return result
def exec_qemunbd(qemunbd_args):
"""
Execute a qemu-nbd command.
Parameters
----------
qemunbd_args: list
The list of arguments for qemu-nbd.
Returns
-------
int: 0 on success, raise exception otherwise.
"""
cmd = ['qemu-nbd'] + qemunbd_args
_logger.debug('__ Running %s' % cmd)
pause_msg(cmd)
try:
return run_call_cmd(cmd)
except Exception as e:
_logger.error(' %s command failed: %s' % (cmd, str(e)))
raise OciMigrateException('\n%s command failed: %s' % (cmd, str(e)))
def exec_blkid(blkid_args):
"""
Run a blkid command.
Parameters
----------
blkid_args: list
The argument list for the blkid command.
Returns
-------
dict: blkid return value on success, None otherwise.
"""
cmd = ['blkid'] + blkid_args
_logger.debug('__ Running %s' % cmd)
try:
pause_msg('test nbd devs')
blkid_res = run_popen_cmd(cmd).decode('utf-8')
_logger.debug('success\n%s' % blkid_res)
return blkid_res
except Exception as e:
_logger.error(' %s failed: %s' % (cmd, str(e)))
return None
def exec_vgchange(changecmd):
"""
Execute vgchange command.
Parameters
----------
changecmd: list
Parameters for the vgchange command.
Returns
-------
str: vgchange output.
"""
cmd = ['vgchange'] + changecmd
_logger.debug('__ Running %s' % cmd)
pause_msg(cmd)
try:
output = run_popen_cmd(cmd).decode('utf-8')
_logger.debug('vgchange result:\n%s' % output)
return output
except Exception as e:
_logger.critical(' Failed to execute %s: %s' % (cmd, str(e)))
raise OciMigrateException('Failed to execute %s: %s' % (cmd, str(e)))
def exec_lsblk(lsblk_args):
"""
Run an lsblk command.
Parameters
----------
lsblk_args: list
The argument list for the blkid command.
Returns
-------
dict: blkid return value on success, None otherwise.
"""
cmd = ['lsblk'] + lsblk_args
_logger.debug('__ Running %s' % cmd)
pause_msg(cmd)
try:
lsblk_res = run_popen_cmd(cmd).decode('utf-8')
_logger.debug('Success\n%s' % lsblk_res)
return lsblk_res
except Exception as e:
_logger.error(' %s failed: %s' % (cmd, str(e)))
raise OciMigrateException('%s failed: %s' % (cmd, str(e)))
def mount_lvm2(devname):
"""
Create the mountpoints /mnt/<last part of lvm partitions> and mount the
partitions on those mountpoints, if possible.
Parameters
----------
devname: str
The full path of the device
Returns
-------
list: The list of mounted partitions.
?? need to collect lvm2 list this way??
"""
_logger.debug('__ Running mount lvm2 %s' % devname)
try:
_, nbcols = terminal_dimension()
mountwait = ProgressBar(int(nbcols),
0.2,
progress_chars=['mounting lvm'])
mountwait.start()
#
# physical volumes
if system_tools.exec_pvscan(['--cache'], devname):
_logger.debug('pvscan %s succeeded' % devname)
else:
_logger.critical(' pvscan %s failed' % devname)
#
pause_msg('pvscan test')
#
# volume groups
if system_tools.exec_vgscan(['--verbose']):
_logger.debug('vgscan succeeded')
else:
_logger.critical(' vgscan failed')
#
pause_msg('vgscan test')
#
# logical volumes
vgs = new_volume_groups()
if bool(vgs):
_logger.debug('lvscan succeeded: %s' % vgs)
else:
_logger.critical(' lvscan failed')
#
pause_msg('lvscan test')
#
# make available
vgchange_args = ['--activate', 'y']
vgchange_res = system_tools.exec_vgchange(vgchange_args)
_logger.debug('vgchange:\n%s' % vgchange_res)
#
pause_msg('vgchange_res test')
vgfound = False
if vgchange_res is not None:
for resline in vgchange_res.splitlines():
_logger.debug('vgchange line: %s' % resline)
for vg in list(vgs.keys()):
if vg in resline:
_logger.debug('vgfound set to True')
vgfound = True
else:
_logger.debug('vg %s not in l' % vg)
_logger.debug('vgchange: %s found: %s' % (vgchange_res, vgfound))
#
# for the sake of testing
pause_msg('vgchange_res test')
else:
_logger.critical(' vgchange failed')
return vgs
except Exception as e:
_logger.critical(' Mount lvm %s failed: %s' % (devname, str(e)))
raise OciMigrateException('Mount lvm %s failed: %s' %
(devname, str(e)))
finally:
if system_tools.is_thread_running(mountwait):
mountwait.stop()
def a20_install_extra_pkgs(self):
"""
Install required and useful packages for OL-type installs, read the
list of packages from oci-migrate-conf.yaml, ol_os_packages_to_install.
Returns
-------
bool: True on success, False otherwise.
"""
def get_ol_package_list():
"""
Retrieve the list of packages to install from oci-migrate-config file.
Returns
-------
list: list of package names to install.
"""
_logger.debug('Collection list of packages.')
try:
pkg_list = get_config_data('ol_os_packages_to_install')
_logger.debug('Package list: %s' % pkg_list)
return pkg_list
except Exception as e:
_logger.warning('Failed to find a list of packages: %s' % str(e))
return False
_logger.debug('__ Installing extra packages.')
#
# collect packages to install
packages = get_ol_package_list()
if not bool(packages):
_logger.debug('No extra packages to install.')
return True
#
#
try:
#
# set current nameserver config.
if system_tools.set_nameserver():
_logger.debug('Updating nameserver info succeeded.')
else:
_logger.error(' Failed to update nameserver info.')
#
# get package manipulation tool.
pkg_mgr = self.exec_yum \
if self.package_tool['pkg_mgr'] == 'yum' else self.exec_dnf
#
# install packages
for pkg in packages:
#
# verify if the package is available, the correct channel enabled.
rpmlist = pkg_mgr(self.package_tool['package_available'] + [pkg])
pkg_present = False
for lline in rpmlist.splitlines():
_logger.debug('%s' % lline)
if pkg in lline:
_logger.debug('The rpm %s is available.' % pkg)
pkg_present = True
break
if not pkg_present:
_logger.error(' The rpm %s is missing.' % pkg)
migrate_data.migrate_preparation = False
migrate_data.migrate_non_upload_reason += \
'\n The rpm package %s is missing from ' \
'the yum repository.' % pkg
return False
else:
installoutput = \
pkg_mgr(self.package_tool['package_install'] + [pkg])
_logger.debug('Successfully installed pkg %s:\n%s'
% (pkg, installoutput))
migrate_tools.result_msg(msg='Installed %s.' % pkg,
result=False)
pause_msg(msg='Installed %s here, or not.' % pkg,
pause_flag='_OCI_CHROOT')
#
# restore nameserver data
if system_tools.restore_nameserver():
_logger.debug('Restoring nameserver info succeeded.')
else:
_logger.error(' Failed to restore nameserver info.')
except Exception as e:
errmsg = 'Failed to install one or more packages of ' \
'%s:\n%s' % (packages, str(e))
_logger.critical(' %s' % errmsg)
migrate_tools.error_msg('%s' % errmsg)
migrate_data.migrate_preparation = False
migrate_data.migrate_non_upload_reason += '\n %s' % errmsg
return False
return True
def a20_install_extra_pkgs(self):
"""
Install required and useful packages for OL-type installs, read the
list of packages from oci-migrate-conf.yaml, ol_os_packages_to_install.
Returns
-------
bool: True on success, False otherwise.
"""
def get_ubuntu_package_list():
"""
Retrieve the list of packages to install from oci-migrate-config file.
Returns
-------
list: list of package names to install.
"""
_logger.debug('Collection list of packages.')
try:
pkg_list = get_config_data('ubuntu_os_packages_to_install_apt')
if not bool(pkg_list):
_logger.debug('apt package list is empty.')
return False
_logger.debug('Package list: %s', pkg_list)
return pkg_list
except Exception as e:
_logger.warning('Failed to find a list of packages: %s',
str(e))
return False
_logger.debug('Installing extra packages.')
packages = get_ubuntu_package_list()
if not bool(packages):
_logger.debug('No extra packages to install.')
return True
try:
#
# set current nameserver config.
if system_tools.set_nameserver():
_logger.debug('Updating nameserver info succeeded.')
else:
_logger.error(' Failed to update nameserver info.')
#
# update package list
update_output = self._exec_apt(['update'])
_logger.debug('Successfully updated package list.')
#
# install packages
for pkg in packages:
#
# verify if the package is available.
pkg_present = False
deblist = self._exec_apt(['list', pkg])
for lline in deblist.splitlines():
_logger.debug('%s', lline)
if pkg in lline:
_logger.debug('The deb package %s is available.', pkg)
pkg_present = True
break
if not pkg_present:
_logger.debug('The deb package %s is missing.', pkg)
migrate_data.migrate_preparation = False
migrate_data.migrate_non_upload_reason +=\
'\n The deb package %s is missing from ' \
'the repository.' % pkg
return False
installoutput = self._exec_apt(['install', '-y', pkg])
_logger.debug('Successfully installed %s:\n%s', pkg,
installoutput)
pause_msg(msg='Installed %s here, or not.' % pkg,
pause_flag='_OCI_CHROOT')
if system_tools.restore_nameserver():
_logger.debug('Restoring nameserver info succeeded.')
else:
_logger.error(' Failed to restore nameserver info.')
except Exception as e:
_logger.critical(
' Failed to install one or more packages of %s:\n%s',
packages, str(e))
error_msg('Failed to install one or more packages of %s:\n%s' %
(packages, str(e)))
migrate_data.migrate_non_upload_reason += \
'\n Failed to install on or more packages ' \
'of %s: %s' % (packages, str(e))
return False
return True
def main():
"""
Main
Returns
-------
int
0 on success, 1 otherwise.
"""
#
# set locale
lc_all_to_set = get_config_data('lc_all')
os.environ['LC_ALL'] = "%s" % lc_all_to_set
_logger.debug('Locale set to %s' % lc_all_to_set)
#
# python version
pythonver = sys.version_info[0]
_logger.debug('Python version is %s' % pythonver)
#
# parse the commandline
args = parse_args()
#
# Operator needs to be root.
if system_tools.is_root():
_logger.debug('User is root.')
else:
exit_with_msg(' *** ERROR *** You must run this program with root '
'privileges')
#
# Verbose mode is False by default
migrate_data.verbose_flag = args.verbose_flag
_logger.debug('Verbose level set to %s' % migrate_data.verbose_flag)
#
# Yes flag
migrate_data.yes_flag = args.yes_flag
_logger.debug('Answer to yes/no questions supposed to be yes always.')
#
# collect and save configuration data
migrate_data.oci_image_migrate_config = get_config_data('*')
#
try:
#
# input image
if args.input_image:
imagepath = args.input_image.name
resultfilename = get_config_data('resultfilepath') + \
'_' + \
os.path.splitext(os.path.basename(imagepath))[0] \
+ '.res'
migrate_data.resultfilename = resultfilename
migrate_tools.result_msg(msg='\n Running %s at %s\n'
% ((os.path.basename(sys.argv[0])
+ ' '
+ ' '.join(sys.argv[1:])),
time.ctime()),
flags='w',
result=True)
else:
raise OciMigrateException('Missing argument: input image path.')
#
# Import the 'format' modules and collect the format data
supported_formats = migrate_tools.import_formats()
if not bool(supported_formats):
exit_with_msg(' *** ERROR *** No image format modules found')
if migrate_data.verbose_flag:
show_supported_formats_data(supported_formats)
#
# Check the utilities installed.
util_list, missing_list = test_helpers()
_logger.debug('%s' % util_list)
if migrate_data.verbose_flag:
show_utilities(util_list, missing_list)
if missing_list:
raise OciMigrateException('%s needs packages %s installed.\n'
% (sys.argv[0], missing_list))
#
# if qemu-img is used, the minimal version is 2
qemuversioninfo = qemu_img_version()
if qemuversioninfo[1] < 2:
raise OciMigrateException('Minimal version of qemu-img is 2, '
'%s found.' % qemuversioninfo[0])
else:
_logger.debug('release data ok')
#
# Get the nameserver definition
if system_tools.get_nameserver():
migrate_tools.result_msg(msg='nameserver %s identified.'
% migrate_data.nameserver, result=False)
_logger.debug('Nameserver identified as %s' % migrate_data.nameserver)
else:
migrate_tools.error_msg('Failed to identify nameserver, using %s, '
'but might cause issues.'
% migrate_data.nameserver)
except Exception as e:
_logger.error('*** ERROR *** %s\n' % str(e))
exit_with_msg(' *** ERROR *** %s\n' % str(e))
#
# More on command line arguments.
#
# initialise output
migrate_tools.result_msg(msg='Results are written to %s.'
% migrate_data.resultfilename, result=True)
migrate_tools.result_msg(msg='Input image: %s' % imagepath, result=True)
#
# Verify if readable.
fn_magic = migrate_tools.get_magic_data(imagepath)
if fn_magic is None:
exit_with_msg('*** ERROR *** An error occured while trying to read '
'magic number of File %s.' % imagepath)
else:
pause_msg('Image Magic Number: %s' % fn_magic)
_logger.debug('Magic number %s successfully read' % fn_magic)
#
# Verify if image type is supported.
_logger.debug('Magic number of %s is %s' % (imagepath, fn_magic))
if fn_magic not in supported_formats:
exit_with_msg('*** ERROR *** Image type %s is not recognised.' % fn_magic)
else:
_logger.debug('Image type recognised.')
#
# Get the correct class.
imageclazz = supported_formats[fn_magic]
migrate_tools.result_msg(msg='Type of image %s identified as %s'
% (imagepath, imageclazz['name']), result=True)
pause_msg('Image type is %s' % imageclazz['name'])
#
# Locate the class and module
imageclassdef = getattr(sys.modules['oci_utils.migrate.image_types.%s'
% supported_formats[fn_magic]['name']],
imageclazz['clazz'])
image_object = imageclassdef(imagepath)
#
# Local volume groups.
vgs_result = system_tools.exec_vgs_noheadings()
migrate_data.local_volume_groups = \
vgs_result if bool(vgs_result) \
else []
_logger.debug('Workstation volume groups: %s'
% migrate_data.local_volume_groups)
#
# Rename local volume groups
if bool(migrate_data.local_volume_groups):
rename_msg = '\n The workstation has logical volumes defined. To avoid ' \
'duplicates, the \n logical volume groups will be temporary' \
' renamed to a hexadecimal uuid.\n If you are sure the ' \
'image to be uploaded does not contain logical volumes,\n' \
' or there are no conflicting volume group names, '\
'the rename can be skipped\n\n Keep the volume group names?'
if not read_yn(rename_msg,
waitenter=True,
suppose_yes=migrate_data.yes_flag):
if migrate_tools.verify_local_fstab():
fstab_msg = '\n The fstab file on this workstation seems to ' \
'contain device references\n using /dev/mapper ' \
'devices. The volume group names on this ' \
'workstation\n will be renamed temporarily. ' \
'/dev/mapper devices referring to logical volumes\n' \
' can create problems in this context. To avoid ' \
'this situation\n exit now and modify the ' \
'device specification to LABEL or UUID.\n\n Continue?'
if not read_yn(fstab_msg,
waitenter=True,
suppose_yes=migrate_data.yes_flag):
exit_with_msg('Exiting')
_logger.debug('Rename local volume groups to avoid conflicts.')
vgrename_res = system_tools.exec_rename_volume_groups(
migrate_data.local_volume_groups, 'FORWARD')
migrate_data.local_volume_group_rename = True
else:
_logger.debug('fstab file has no /dev/mapper devices.')
else:
_logger.debug('Not renaming the volume groups.')
_ = system_tools.reset_vg_list(migrate_data.local_volume_groups)
else:
_logger.debug('No local volume groups, no conflicts.')
#
# Generic data collection
try:
imgres, imagedata = collect_image_data(image_object)
if migrate_data.verbose_flag:
migrate_tools.show_image_data(image_object)
if imgres:
_logger.debug('Image processing succeeded.')
else:
_logger.critical(' Image processing failed.', exc_info=False)
#
if imagedata:
_logger.debug('%s passed.' % imagepath)
else:
_logger.critical(' %s failed.' % imagepath, exc_info=False)
except Exception as e:
_logger.critical(' %s failed: %s' % (imagepath, str(e)))
exit_with_msg('*** ERROR *** Problem detected during investigation of '
'the image %s: %s, exiting.' % (imagepath, str(e)))
#
# Restore volume group names.
if migrate_data.local_volume_group_rename:
_logger.debug('Restore local volume groups.')
vgrename_res = system_tools.exec_rename_volume_groups(
migrate_data.local_volume_groups, 'BACKWARD')
else:
_logger.debug('No local volume group names to restore.')
#
# passed prerequisites and changes?
prereq_passed = True
#
# Image type specific prerequisites
prereq_msg = ''
sup, msg = image_object.type_specific_prereq_test()
if sup:
migrate_tools.result_msg(msg='%s' % msg, result=True)
else:
prereq_passed = False
prereq_msg = msg
#
# Generic prerequisites verification
try:
gen_prereq, msg = image_object.generic_prereq_check()
if gen_prereq:
prereq_msg += '\n %s passed the generic preqrequisites.' % imagepath
else:
prereq_passed = False
prereq_msg += msg
#
if imgres:
prereq_msg += '\n\n %s data collection and processing succeeded.' \
% imagepath
else:
prereq_passed = False
#
if prereq_passed:
migrate_tools.result_msg(msg=prereq_msg, result=True)
if imagedata['boot_type'] == 'BIOS':
migrate_tools.result_msg(msg='\n Boot type is BIOS, '
'use launch_mode PARAVIRTUALIZED '
'(or EMULATED) at import.',
result=True)
elif imagedata['boot_type'] == 'UEFI':
migrate_tools.result_msg(msg='\n Boot type is UEFI, '
'use launch_mode NATIVE '
'(or EMULATED) at import.',
result=True)
else:
raise OciMigrateException('Something wrong checking '
'the boot type')
else:
prereq_msg += '\n\n %s processing failed, check the logfile ' \
'and/or set environment variable _OCI_UTILS_DEBUG.' \
% imagepath
raise OciMigrateException(prereq_msg)
except Exception as e:
exit_with_msg('*** ERROR *** %s' % str(e))
#
# While prerequisite check did not hit a fatal issue, there might be
# situations where upload should not proceed.
if not migrate_data.migrate_preparation:
exit_with_msg('*** ERROR *** Unable to proceed with uploading image '
'to Oracle Cloud Infrastructure: %s'
% migrate_data.migrate_non_upload_reason)
else:
migrate_tools.result_msg('Successfully verified and processed image %s '
'and is ready for upload.' % imagepath)
return 0
