def bucket_exists(bucket_name): """ Verify if bucket_name exits. Parameters ---------- bucket_name: str The bucket_name. Returns ------- dict: The bucket on success, raise an exception otherwise """ cmd = [ 'oci', 'os', 'object', 'list', '--all', '--bucket-name', bucket_name ] _logger.debug('__ Running %s.', cmd) pause_msg(cmd) try: bucket_result = json.loads( system_tools.run_popen_cmd(cmd)['output'].decode('utf-8')) _logger.debug('Result: \n%s', bucket_result) return bucket_result except Exception as e: _logger.debug( 'Bucket %s does not exists or the authorisation is missing: %s.', bucket_name, str(e)) raise OciMigrateException( 'Bucket %s does not exists or the authorisation is missing:' % bucket_name) from e
def exec_vgscan(vgscan_args): """ Scan the system for (new) volume groups. Parameters ---------- vgscan_args: list list of strings, arguments for vgscan Returns ------- bool: True on success, raises an exeception on failure. """ cmd = ['vgscan'] + vgscan_args _logger.debug('__ Executing %s' % cmd) pause_msg(cmd) try: output = run_popen_cmd(cmd).decode('utf-8') _logger.debug('Volume groups scanned:\n%s' % str(output)) return True except Exception as e: # # vgscan failed _logger.critical(' Failed to scan for volume groups: %s' % str(e)) raise OciMigrateException('Failed to scan for volume groups: %s' % str(e))
def mount_partition(devname, mountpoint=None): """ Create the mountpoint /mnt/<last part of device specification> and mount a partition on on this mountpoint. Parameters ---------- devname: str The full path of the device. mountpoint: str The mountpoint, will be generated if not provided. Returns ------- str: The mounted partition on Success, None otherwise. """ _logger.debug('__ Mount partition %s.', devname) # # create mountpoint /mnt/<devname> if not specified. if mountpoint is None: mntpoint = migrate_data.loopback_root + '/' + devname.rsplit('/')[-1] _logger.debug('Loopback mountpoint: %s', mntpoint) try: if system_tools.exec_mkdir(mntpoint): _logger.debug('Mountpoint: %s created.', mntpoint) except Exception as e: _logger.critical(' Failed to create mountpoint %s: %s', mntpoint, str(e)) raise OciMigrateException('Failed to create mountpoint %s:' % mntpoint) from e else: mntpoint = mountpoint # # actual mount cmd = ['mount', devname, mntpoint] pause_msg(cmd, pause_flag='_OCI_MOUNT') _, nbcols = terminal_dimension() try: mountpart = ProgressBar(nbcols, 0.2, progress_chars=['mount %s' % devname]) mountpart.start() _logger.debug('command: %s', cmd) cmdret = system_tools.run_call_cmd(cmd) if cmdret == 0: _logger.debug('%s mounted on %s.', devname, mntpoint) return mntpoint raise Exception('Mount %s failed: %d' % (devname, cmdret)) except Exception as e: # # mount failed, need to remove mountpoint. _logger.critical(' Failed to mount %s, missing driver, filesystem corruption...: %s', devname, str(e)) if mountpoint is None: if system_tools.exec_rmdir(mntpoint): _logger.debug('%s removed', mntpoint) else: _logger.critical(' Failed to remove mountpoint %s', mntpoint) finally: if system_tools.is_thread_running(mountpart): mountpart.stop() return None
def mount_pseudo(rootdir): """ Remount proc, sys and dev. Parameters ---------- rootdir: str The mountpoint of the root partition. Returns ------- list: The list of new mountpoints on success, None otherwise. """ pseudodict = {'proc': ['-t', 'proc', 'none', '%s/proc' % rootdir], 'dev': ['-o', 'bind', '/dev', '%s/dev' % rootdir], 'sys': ['-o', 'bind', '/sys', '%s/sys' % rootdir]} pseudomounts = [] _logger.debug('__ Mounting: %s', pseudodict) for dirs, cmd_par in list(pseudodict.items()): cmd = ['mount'] + cmd_par _logger.debug('Mounting %s', dirs) pause_msg(cmd, pause_flag='_OCI_MOUNT') try: _logger.debug('Command: %s', cmd) cmdret = run_call_cmd(cmd) _logger.debug('%s : %d', cmd, cmdret) if cmdret != 0: _logger.error(' Failed to %s', cmd) raise Exception('%s Failed: %d' % (cmd, cmdret)) pseudomounts.append(cmd_par[3]) except Exception as e: _logger.critical(' Failed to %s: %s', cmd, str(e)) raise OciMigrateException('Failed to %s:' % cmd) from e return pseudomounts
def exec_lvscan(lvscan_args): """ Scan the system for logical volumes. Parameters ---------- lvscan_args: list list of strings, arguments for lvscan Returns ------- list: A list of strings, the output of lvscan --verbose on success, raises an exeception on failure. """ cmd = ['lvscan'] + lvscan_args _logger.debug('__ Running: %s' % cmd) pause_msg(cmd) try: _logger.debug('command: %s' % cmd) output = run_popen_cmd(cmd).decode('utf-8') _logger.debug('Logical volumes scanned:\n%s' % str(output)) return output.splitlines() except Exception as e: # # lvscan failed _logger.critical(' Failed to scan for logical volumes: %s' % str(e)) raise OciMigrateException('Failed to scan for logical volume: %s' % str(e))
def upload_image(imgname, bucket_name, ociname): """ Upload the validated and updated image imgname to the OCI object storage bucket_name as ociname. Parameters ---------- imgname: str The on-premise custom image. bucket_name: str The OCI object storage name. ociname: The OCI image name. Returns ------- bool: True on success, raises an exception otherwise. """ cmd = ['oci', 'os', 'object', 'put', '--bucket-name', bucket_name, '--file', imgname, '--name', ociname, '--part-size', '100', '--parallel-upload-count', '6'] _logger.debug('__ Running %s', cmd) pause_msg(cmd) try: upload_result = system_tools.run_popen_cmd(cmd)['output'].decode('utf-8') _logger.debug('Successfully uploaded %s to %s as %s: %s.', imgname, bucket_name, ociname, upload_result) return True except Exception as e: _logger.critical(' Failed to upload %s to object storage %s as %s: %s.', imgname, bucket_name, ociname, str(e)) raise OciMigrateException('Failed to upload %s to object storage %s as %s:' % (imgname, bucket_name, ociname)) from e
def mount_fs(mountpoint): """ Mount a filesystem specified in fstab, by mountpoint only. Parameters ---------- mountpoint: str The mountpoint. Returns ------- bool: True on success, False otherwise """ cmd = ['mount', mountpoint] pause_msg(cmd) _logger.debug('__ Mounting %s' % mountpoint) try: _, nbcols = terminal_dimension() mountwait = ProgressBar(nbcols, 0.2, progress_chars=['mounting %s' % mountpoint]) mountwait.start() _logger.debug('Command: %s' % cmd) cmdret = run_call_cmd(cmd) _logger.debug('%s returned %d' % (cmd, cmdret)) if cmdret == 0: return True else: raise Exception('%s failed: %d' % (cmd, cmdret)) except Exception as e: _logger.error(' Failed to %s: %s' % (cmd, str(e))) return False finally: if is_thread_running(mountwait): mountwait.stop()
def unmount_something(mountpoint): """ Unmount. Parameters ---------- mountpoint: str The mountpoint. Returns ------- bool: True on success, False otherwise. """ _logger.debug('__ Unmount %s.' % mountpoint) if os.path.ismount(mountpoint): _logger.debug('%s is a mountpoint.' % mountpoint) else: _logger.debug('%s is not a mountpoint, quitting' % mountpoint) return True # cmd = ['umount', mountpoint] pause_msg(cmd) try: _logger.debug('command: %s' % cmd) cmdret = run_call_cmd(cmd) _logger.debug('%s : %d' % (cmd, cmdret)) if cmdret != 0: raise Exception('%s failed: %d' % (cmd, cmdret)) except Exception as e: _logger.error(' Failed to %s: %s' % (cmd, str(e))) return False return True
def b20_install_cloud_agent(self): """ Install the oracle cloud agent. Returns ------- bool: True on success, False otherwise. (always True as failing to install the oracle cloud agent is not fatal. """ _logger.debug('__ Install oracle cloud agent.') if bool(migrate_data.oracle_cloud_agent_location): _logger.debug('oracle cloud agent present: %s', migrate_data.oracle_cloud_agent_location) else: _logger.debug('No oracle cloud agent package present, skipping.') return True # # get package manipulation tool. pkg_mgr = self._exec_yum \ if self.package_tool['pkg_mgr'] == 'yum' else self._exec_dnf # # install rpm oracle_cloud_agent_rpm = migrate_data.oracle_cloud_agent_location simple_rpm = oracle_cloud_agent_rpm.split('/')[-1] try: install_output = pkg_mgr( self.package_tool['package_localinstall'] + [oracle_cloud_agent_rpm]) _logger.debug('Successfully installed pkg %s:\n%s', simple_rpm, install_output) migrate_tools.result_msg(msg='Installed %s.' % simple_rpm, result=False) pause_msg('cloud agent', pause_flag='_OCI_AGENT') except Exception as e: _logger.warning('Failed to install %s: %s', simple_rpm, str(e)) return True
def exec_vgs_noheadings(): """ List the local volume group and generates a new (temporary) name as a hex UUID. Returns ------- list: list of lists [orginal volume group name, new volume group name]. """ cmd = ['vgs', '--noheadings'] _logger.debug('__ Executing %s.' % cmd) pause_msg(cmd) vg_list = list() try: vgs_response = run_popen_cmd(cmd) output = vgs_response.decode('utf-8').splitlines() if bool( vgs_response) else b'' if bool(output): for vg_record in output: if len(vg_record) > 0: vg_list.append([vg_record.split()[0], uuid.uuid4().hex]) _logger.debug('Volume groups found: %s' % vg_list) return vg_list except Exception as e: _logger.critical(' Failed to list current volume groups: %s' % str(e))
def exec_pvscan(pvscan_args, devname=None): """ Update the lvm cache. Parameters ---------- pvscan_args: list List of strings, arguments for pvscan devname: str Device name to scan. Returns ------- bool: True on success, raises an exception on failure. """ _logger.debug('__ Running pvscan %s' % pvscan_args) cmd = ['pvscan'] + pvscan_args if devname is not None: cmd.append(devname) pause_msg(cmd) try: _logger.debug('command: %s' % cmd) cmdret = run_call_cmd(cmd) _logger.debug('Physical volumes scanned on %s: %d' % (devname, cmdret)) if cmdret != 0: _logger.error(' Physical volume scan failed.') raise Exception('Physical volume scan failed.') return True except Exception as e: # # pvscan failed _logger.critical(' Failed to scan %s for physical volumes: %s' % (devname, str(e))) raise OciMigrateException('Failed to scan %s for physical ' 'volumes: %s' % (devname, str(e)))
def get_cloud_agent_if_relevant(root_part, os_type, major_release): """ Download oracle cloud agent for os where this cannot be achieved via standard install channels. Parameters ---------- root_part: str The root partition of the image as currently mounted. os_type: str The os type id as set in the os-release file. major_release: str The major release info. Returns ------- bool: always True """ _logger.debug('__ Collecting the oracle_cloud_agent package if relevant.') _logger.debug('root partition: %s' % root_part) _logger.debug('os type: %s' % os_type) _logger.debug('major release: %s' % major_release) pause_msg('cloud agent', pause_flag='_OCI_AGENT') try: if os_type in get_config_data('ol_os_for_cloud_agent'): _logger.debug('OS is OL-type os.') el_tag = get_config_data('ol_version_id_dict')[major_release] _logger.debug('el tag: %s' % el_tag) cloud_agent_url = get_url_data_from_base(el_tag) _logger.debug('cloud agent url: %s' % cloud_agent_url) if cloud_agent_url is None: _logger.debug('URL with oracle cloud agent package not found.') else: package_name = os.path.basename(cloud_agent_url) destination_dir = get_config_data( 'ol_os_oracle_cloud_agent_store') destination = os.path.join(root_part, destination_dir.lstrip('/'), package_name.lstrip('/')) _logger.debug( 'Destination for oracle cloud agent package: %s' % destination) _ = get_file_from_url(cloud_agent_url, destination) migrate_data.oracle_cloud_agent_location = \ os.path.join('/', destination_dir.lstrip('/'), package_name.lstrip('/')) _logger.debug('cloud agent location: %s' % migrate_data.oracle_cloud_agent_location) else: _logger.debug('This operation is not relevant here.') return True except Exception as e: _logger.debug('Installation of the oracle cloud agent failed: %s' % str(e)) pause_msg('cloud agent', pause_flag='_OCI_AGENT') return True
def a10_remove_cloud_init(self): """ Remove cloud-init package and configuration Returns ------- bool: True on success, False otherwise. """ _logger.debug('__ Remove existing cloud_init config.') try: # # write 90_dpkg.cfg dpkg_cfg_path = '/etc/cloud/cloud.cfg.d' if os.path.exists(dpkg_cfg_path): with open(dpkg_cfg_path + '/90_dpkg.cfg', 'w') as f: f.write('datasource_list: [None]\n') _logger.debug('%s (re)written.', dpkg_cfg_path) else: _logger.debug('%s does not exist.', dpkg_cfg_path) # # remove cloud-init package purge_output = self._exec_apt(['purge', 'cloud-init', '-y']) _logger.debug('cloud-init purged: %s', purge_output) # # remove /etc/cloud cloud_cfg_path = '/etc/cloud' backup_cloud_path = system_tools.exec_rename(cloud_cfg_path) if bool(backup_cloud_path): _logger.debug('%s renamed to %s', cloud_cfg_path, backup_cloud_path) # # remove /var/lib/cloud var_lib_cloud_path = '/var/lib/cloud' backup_var_lib_cloud_path = system_tools.exec_rename( var_lib_cloud_path) if bool(backup_var_lib_cloud_path): _logger.debug('%s renamed to %s', var_lib_cloud_path, backup_var_lib_cloud_path) # # remove logs cloud_init_log = '/var/log/cloud-init.log' backup_cloud_init_log = system_tools.exec_rename(cloud_init_log) if bool(cloud_init_log): _logger.debug('%s renamed to %s', cloud_init_log, backup_cloud_init_log) # pause_msg(msg='cloud-init removed', pause_flag='_OCI_CHROOT') return True except Exception as e: _logger.warning( 'Failed to purge cloud-init completely which might cause issues ' 'at instance creation: %s', str(e)) return False
def exec_parted(devname): """ Collect data about the device on the image using the parted utility. Parameters ---------- devname: str The device name. Returns ------- dict: The device data from parted utility on success, None otherwise. """ cmd = ['parted', devname, 'print'] _logger.debug('__ Running %s' % cmd) pause_msg(cmd) try: result = run_popen_cmd(cmd).decode('utf-8') _logger.debug('parted: %s' % result) device_data = dict() device_data['Partition List'] = list() for devx in result.splitlines(): _logger.debug('%s - %d' % (devx, len(devx))) if 'Model' in devx: device_data['Model'] = devx.split(':')[1] _logger.debug('Model %s' % device_data['Model']) elif 'Disk Flags' in devx: device_data['Disk Flags'] = devx.split(':')[1] _logger.debug('Disk Flags %s' % device_data['Disk Flags']) elif 'Disk' in devx: device_data['Disk'] = devx.split(':')[1] _logger.debug('Disk %s' % device_data['Disk']) elif 'Partition Table' in devx: device_data['Partition Table'] = devx.split(':')[1] _logger.debug('Partition Table %s' % device_data['Partition Table']) elif devx.split(): if devx.split()[0].isdigit(): device_data['Partition List'].append(devx.split()) _logger.debug('Partition: %s' % devx.split()) else: _logger.debug('Ignoring %s' % devx) else: _logger.debug('Ignoring %s' % devx) _logger.debug(device_data) pause_msg(device_data) return device_data except Exception as e: _logger.error(' Failed to collect parted %s device data: %s' % (devname, str(e))) return None
def mount_imgfn(imgname): """ Link vm image with an nbd device. Parameters ---------- imgname: str Full path of the image file. Returns ------- str: Device name on success, raises an exception otherwise. """ # # create nbd devices _logger.debug('__ Running mount image file %s.' % imgname) result_msg(msg='Load nbd') if not system_tools.create_nbd(): raise OciMigrateException('Failed ot load nbd module') else: _logger.debug('nbd module loaded') # # find free nbd device result_msg(msg='Find free nbd device') devpath = system_tools.get_free_nbd() _logger.debug('Device %s is free.' % devpath) # # link img with first free nbd device result_msg(msg='Mount image %s' % imgname, result=True) _, nbcols = terminal_dimension() try: mountwait = ProgressBar(nbcols, 0.2, progress_chars=['mounting image']) mountwait.start() qemucmd = ['-c', devpath, imgname] pause_msg(qemucmd) qemunbd_ret = system_tools.exec_qemunbd(qemucmd) if qemunbd_ret == 0: time.sleep(5) _logger.debug('qemu-nbd %s succeeded' % qemucmd) return devpath else: _logger.critical('\n Failed to create nbd devices: %d' % qemunbd_ret) raise Exception('Failed to create nbd devices: %d' % qemunbd_ret) except Exception as e: _logger.critical('\n Something wrong with creating nbd devices: %s' % str(e)) raise OciMigrateException('Unable to create nbd devices: %s' % str(e)) finally: if system_tools.is_thread_running(mountwait): mountwait.stop()
def unmount_part(devname): """ Unmount a partition from mountpoint from /mnt/<last part of device specification> and remove the mountpoint. Parameters ---------- devname: str The full path of the device. Returns ------- bool True on success, False otherwise. """ mntpoint = migrate_data.loopback_root + '/' + devname.rsplit('/')[-1] cmd = ['umount', mntpoint] _logger.debug('__ Running %s' % cmd) pause_msg(cmd) while True: try: _logger.debug('command: %s' % cmd) cmdret = system_tools.run_call_cmd(cmd) if cmdret == 0: _logger.debug('%s unmounted from %s' % (devname, mntpoint)) # # remove mountpoint if system_tools.exec_rmdir(mntpoint): _logger.debug('%s removed' % mntpoint) return True else: _logger.critical(' Failed to remove mountpoint %s' % mntpoint) raise OciMigrateException( 'Failed to remove mountpoint %s' % mntpoint) else: _logger.critical(' Failed to unmount %s: %d' % (devname, cmdret)) console_msg('Failed to unmount %s, error code %d.\n ' 'Please verify before continuing.' % (devname, cmdret)) retry = read_yn( 'Something prevented to complete %s, please ' 'verify and correct if possible. ' 'Press Y to retry, N to ignore.', waitenter=True) if not retry: break except Exception as e: _logger.critical(' Failed to unmount %s: %s' % (devname, str(e))) return False
def exec_sfdisk(devname): """ Collect the data about the partitions on the image file mounted on the device devname using the sfdisk utility. Parameters ---------- devname: str The device. Returns ------- dict: The partition data with sfdisk results on success, None otherwise. """ cmd = ['sfdisk', '-d', devname] _logger.debug('__ Running %s' % cmd) pause_msg(cmd) try: result = run_popen_cmd(cmd).decode('utf-8') partdata = dict() for devx in result.splitlines(): if devx.startswith(devname): key = devx.split(':')[0].strip() migrate_tools.result_msg(msg='sfdisk partition %s' % key, result=False) thispart = {'start': 0, 'size': 0, 'Id': 0, 'bootable': False} val = devx.split(':')[1].split(',') for it in val: if 'start' in it: x = it.split('=')[1] thispart['start'] = int(x) elif 'size' in it: x = it.split('=')[1] thispart['size'] = int(x) elif 'Id' in it: x = it.split('=')[1] thispart['Id'] = x.strip() elif 'bootable' in it: thispart['bootable'] = True else: _logger.debug('unrecognised item: %s' % val) partdata[key] = thispart _logger.debug(partdata) return partdata except Exception as e: _logger.error(' Failed to collect sfdisk %s partition data: %s' % (devname, str(e))) return None
def exec_ldconfig(): """ Executes ldconfig to update the shared library cache. Returns ------- int: 0 on success, raises an exception otherwise. """ cmd = ['ldconfig'] _logger.debug('__ Running %s' % cmd) try: pause_msg('running ldconfig') return run_call_cmd(cmd) except Exception as e: _logger.error(' %s command failed: %s' % (cmd, str(e))) raise OciMigrateException('\n%s command failed: %s' % (cmd, str(e)))
def a10_remove_cloud_init(self): """ Remove the cloud-init software and the configuration data. Returns ------- bool: True on success, False otherwise. """ _logger.debug('__ Remove existing cloud_init config.') try: pkg_mgr = self._exec_yum \ if self.package_tool['pkg_mgr'] == 'yum' else self._exec_dnf package_erase = self.package_tool['package_remove'] + [ 'cloud-init' ] remove_output = pkg_mgr(package_erase) _logger.debug('cloud-init removed: %s', remove_output) # # remove /etc/cloud cloud_cfg_path = '/etc/cloud' backup_cloud_path = system_tools.exec_rename(cloud_cfg_path) if bool(backup_cloud_path): _logger.debug('%s renamed to %s', cloud_cfg_path, backup_cloud_path) # # remove /var/lib/cloud var_lib_cloud_path = '/var/lib/cloud' backup_var_lib_cloud_path = system_tools.exec_rename( var_lib_cloud_path) if bool(backup_var_lib_cloud_path): _logger.debug('%s renamed to %s', var_lib_cloud_path, backup_var_lib_cloud_path) # # remove logs cloud_init_log = '/var/log/cloud-init.log' backup_cloud_init_log = system_tools.exec_rename(cloud_init_log) if bool(cloud_init_log): _logger.debug('%s renamed to %s', cloud_init_log, backup_cloud_init_log) # pause_msg(msg='cloud-init removed', pause_flag='_OCI_CHROOT') return True except Exception as e: _logger.warning( 'Failed to remove cloud-init completely which might cause issues at instance ' 'creation: %s', str(e)) return False
def unmount_imgfn(devname): """ Unlink a device. Parameters ---------- devname: str The device name Returns ------- bool: True on succes, raise an exception otherwise. """ _logger.debug('__ Unmount %s' % devname) try: # # release device qemucmd = ['-d', devname] pause_msg(qemucmd) qemunbd_ret = system_tools.exec_qemunbd(qemucmd) if qemunbd_ret == 0: _logger.debug('qemu-nbd %s succeeded: %d' % (qemucmd, qemunbd_ret)) else: raise Exception('%s returned %d' % (qemucmd, qemunbd_ret)) # # clear lvm cache, if necessary. if system_tools.exec_pvscan(['--cache']): _logger.debug('lvm cache updated') else: _logger.error(' Failed to clear LVM cache.') raise OciMigrateException('Failed to clear LVM cache.') # # remove nbd module if not system_tools.rm_nbd(): raise OciMigrateException('Failed to remove nbd module.') else: _logger.debug('Successfully removed nbd module.') except Exception as e: _logger.critical(' Something wrong with removing nbd devices: %s' % str(e)) raise OciMigrateException('\nSomething wrong with removing nbd ' 'devices: %s' % str(e)) return True
def exec_rename_volume_groups(vg_list, direction): """ Rename a list of volume groups. Parameters ---------- vg_list: list list of lists [original name, new name] direction: str if FORWARD, rename original name to new name, if BACKWARD from new name to original name. Returns ------- bool: True on success, False otherwise. """ _logger.debug('__ Rename volume group %s.' % vg_list) result = True # for vg_names in vg_list: if direction == 'FORWARD': cmd = ['vgrename', vg_names[0], vg_names[1]] elif direction == 'BACKWARD': cmd = ['vgrename', vg_names[1], vg_names[0]] else: _logger.debug('Invalid argument %s' % direction) return False # pause_msg(cmd) try: _logger.debug('command: %s' % cmd) output = run_popen_cmd(cmd).decode('utf-8') if 'successfully renamed' in output: _logger.debug('%s succeeded' % cmd) else: _logger.debug('%s failed' % cmd) result = False except Exception as e: _logger.debug('Execution of vgrename failed: %s' % str(e)) result = False return result
def exec_qemunbd(qemunbd_args): """ Execute a qemu-nbd command. Parameters ---------- qemunbd_args: list The list of arguments for qemu-nbd. Returns ------- int: 0 on success, raise exception otherwise. """ cmd = ['qemu-nbd'] + qemunbd_args _logger.debug('__ Running %s' % cmd) pause_msg(cmd) try: return run_call_cmd(cmd) except Exception as e: _logger.error(' %s command failed: %s' % (cmd, str(e))) raise OciMigrateException('\n%s command failed: %s' % (cmd, str(e)))
def exec_blkid(blkid_args): """ Run a blkid command. Parameters ---------- blkid_args: list The argument list for the blkid command. Returns ------- dict: blkid return value on success, None otherwise. """ cmd = ['blkid'] + blkid_args _logger.debug('__ Running %s' % cmd) try: pause_msg('test nbd devs') blkid_res = run_popen_cmd(cmd).decode('utf-8') _logger.debug('success\n%s' % blkid_res) return blkid_res except Exception as e: _logger.error(' %s failed: %s' % (cmd, str(e))) return None
def exec_vgchange(changecmd): """ Execute vgchange command. Parameters ---------- changecmd: list Parameters for the vgchange command. Returns ------- str: vgchange output. """ cmd = ['vgchange'] + changecmd _logger.debug('__ Running %s' % cmd) pause_msg(cmd) try: output = run_popen_cmd(cmd).decode('utf-8') _logger.debug('vgchange result:\n%s' % output) return output except Exception as e: _logger.critical(' Failed to execute %s: %s' % (cmd, str(e))) raise OciMigrateException('Failed to execute %s: %s' % (cmd, str(e)))
def exec_lsblk(lsblk_args): """ Run an lsblk command. Parameters ---------- lsblk_args: list The argument list for the blkid command. Returns ------- dict: blkid return value on success, None otherwise. """ cmd = ['lsblk'] + lsblk_args _logger.debug('__ Running %s' % cmd) pause_msg(cmd) try: lsblk_res = run_popen_cmd(cmd).decode('utf-8') _logger.debug('Success\n%s' % lsblk_res) return lsblk_res except Exception as e: _logger.error(' %s failed: %s' % (cmd, str(e))) raise OciMigrateException('%s failed: %s' % (cmd, str(e)))
def mount_lvm2(devname): """ Create the mountpoints /mnt/<last part of lvm partitions> and mount the partitions on those mountpoints, if possible. Parameters ---------- devname: str The full path of the device Returns ------- list: The list of mounted partitions. ?? need to collect lvm2 list this way?? """ _logger.debug('__ Running mount lvm2 %s' % devname) try: _, nbcols = terminal_dimension() mountwait = ProgressBar(int(nbcols), 0.2, progress_chars=['mounting lvm']) mountwait.start() # # physical volumes if system_tools.exec_pvscan(['--cache'], devname): _logger.debug('pvscan %s succeeded' % devname) else: _logger.critical(' pvscan %s failed' % devname) # pause_msg('pvscan test') # # volume groups if system_tools.exec_vgscan(['--verbose']): _logger.debug('vgscan succeeded') else: _logger.critical(' vgscan failed') # pause_msg('vgscan test') # # logical volumes vgs = new_volume_groups() if bool(vgs): _logger.debug('lvscan succeeded: %s' % vgs) else: _logger.critical(' lvscan failed') # pause_msg('lvscan test') # # make available vgchange_args = ['--activate', 'y'] vgchange_res = system_tools.exec_vgchange(vgchange_args) _logger.debug('vgchange:\n%s' % vgchange_res) # pause_msg('vgchange_res test') vgfound = False if vgchange_res is not None: for resline in vgchange_res.splitlines(): _logger.debug('vgchange line: %s' % resline) for vg in list(vgs.keys()): if vg in resline: _logger.debug('vgfound set to True') vgfound = True else: _logger.debug('vg %s not in l' % vg) _logger.debug('vgchange: %s found: %s' % (vgchange_res, vgfound)) # # for the sake of testing pause_msg('vgchange_res test') else: _logger.critical(' vgchange failed') return vgs except Exception as e: _logger.critical(' Mount lvm %s failed: %s' % (devname, str(e))) raise OciMigrateException('Mount lvm %s failed: %s' % (devname, str(e))) finally: if system_tools.is_thread_running(mountwait): mountwait.stop()
def a20_install_extra_pkgs(self): """ Install required and useful packages for OL-type installs, read the list of packages from oci-migrate-conf.yaml, ol_os_packages_to_install. Returns ------- bool: True on success, False otherwise. """ def get_ol_package_list(): """ Retrieve the list of packages to install from oci-migrate-config file. Returns ------- list: list of package names to install. """ _logger.debug('Collection list of packages.') try: pkg_list = get_config_data('ol_os_packages_to_install') _logger.debug('Package list: %s' % pkg_list) return pkg_list except Exception as e: _logger.warning('Failed to find a list of packages: %s' % str(e)) return False _logger.debug('__ Installing extra packages.') # # collect packages to install packages = get_ol_package_list() if not bool(packages): _logger.debug('No extra packages to install.') return True # # try: # # set current nameserver config. if system_tools.set_nameserver(): _logger.debug('Updating nameserver info succeeded.') else: _logger.error(' Failed to update nameserver info.') # # get package manipulation tool. pkg_mgr = self.exec_yum \ if self.package_tool['pkg_mgr'] == 'yum' else self.exec_dnf # # install packages for pkg in packages: # # verify if the package is available, the correct channel enabled. rpmlist = pkg_mgr(self.package_tool['package_available'] + [pkg]) pkg_present = False for lline in rpmlist.splitlines(): _logger.debug('%s' % lline) if pkg in lline: _logger.debug('The rpm %s is available.' % pkg) pkg_present = True break if not pkg_present: _logger.error(' The rpm %s is missing.' % pkg) migrate_data.migrate_preparation = False migrate_data.migrate_non_upload_reason += \ '\n The rpm package %s is missing from ' \ 'the yum repository.' % pkg return False else: installoutput = \ pkg_mgr(self.package_tool['package_install'] + [pkg]) _logger.debug('Successfully installed pkg %s:\n%s' % (pkg, installoutput)) migrate_tools.result_msg(msg='Installed %s.' % pkg, result=False) pause_msg(msg='Installed %s here, or not.' % pkg, pause_flag='_OCI_CHROOT') # # restore nameserver data if system_tools.restore_nameserver(): _logger.debug('Restoring nameserver info succeeded.') else: _logger.error(' Failed to restore nameserver info.') except Exception as e: errmsg = 'Failed to install one or more packages of ' \ '%s:\n%s' % (packages, str(e)) _logger.critical(' %s' % errmsg) migrate_tools.error_msg('%s' % errmsg) migrate_data.migrate_preparation = False migrate_data.migrate_non_upload_reason += '\n %s' % errmsg return False return True
def a20_install_extra_pkgs(self): """ Install required and useful packages for OL-type installs, read the list of packages from oci-migrate-conf.yaml, ol_os_packages_to_install. Returns ------- bool: True on success, False otherwise. """ def get_ubuntu_package_list(): """ Retrieve the list of packages to install from oci-migrate-config file. Returns ------- list: list of package names to install. """ _logger.debug('Collection list of packages.') try: pkg_list = get_config_data('ubuntu_os_packages_to_install_apt') if not bool(pkg_list): _logger.debug('apt package list is empty.') return False _logger.debug('Package list: %s', pkg_list) return pkg_list except Exception as e: _logger.warning('Failed to find a list of packages: %s', str(e)) return False _logger.debug('Installing extra packages.') packages = get_ubuntu_package_list() if not bool(packages): _logger.debug('No extra packages to install.') return True try: # # set current nameserver config. if system_tools.set_nameserver(): _logger.debug('Updating nameserver info succeeded.') else: _logger.error(' Failed to update nameserver info.') # # update package list update_output = self._exec_apt(['update']) _logger.debug('Successfully updated package list.') # # install packages for pkg in packages: # # verify if the package is available. pkg_present = False deblist = self._exec_apt(['list', pkg]) for lline in deblist.splitlines(): _logger.debug('%s', lline) if pkg in lline: _logger.debug('The deb package %s is available.', pkg) pkg_present = True break if not pkg_present: _logger.debug('The deb package %s is missing.', pkg) migrate_data.migrate_preparation = False migrate_data.migrate_non_upload_reason +=\ '\n The deb package %s is missing from ' \ 'the repository.' % pkg return False installoutput = self._exec_apt(['install', '-y', pkg]) _logger.debug('Successfully installed %s:\n%s', pkg, installoutput) pause_msg(msg='Installed %s here, or not.' % pkg, pause_flag='_OCI_CHROOT') if system_tools.restore_nameserver(): _logger.debug('Restoring nameserver info succeeded.') else: _logger.error(' Failed to restore nameserver info.') except Exception as e: _logger.critical( ' Failed to install one or more packages of %s:\n%s', packages, str(e)) error_msg('Failed to install one or more packages of %s:\n%s' % (packages, str(e))) migrate_data.migrate_non_upload_reason += \ '\n Failed to install on or more packages ' \ 'of %s: %s' % (packages, str(e)) return False return True
def main(): """ Main Returns ------- int 0 on success, 1 otherwise. """ # # set locale lc_all_to_set = get_config_data('lc_all') os.environ['LC_ALL'] = "%s" % lc_all_to_set _logger.debug('Locale set to %s' % lc_all_to_set) # # python version pythonver = sys.version_info[0] _logger.debug('Python version is %s' % pythonver) # # parse the commandline args = parse_args() # # Operator needs to be root. if system_tools.is_root(): _logger.debug('User is root.') else: exit_with_msg(' *** ERROR *** You must run this program with root ' 'privileges') # # Verbose mode is False by default migrate_data.verbose_flag = args.verbose_flag _logger.debug('Verbose level set to %s' % migrate_data.verbose_flag) # # Yes flag migrate_data.yes_flag = args.yes_flag _logger.debug('Answer to yes/no questions supposed to be yes always.') # # collect and save configuration data migrate_data.oci_image_migrate_config = get_config_data('*') # try: # # input image if args.input_image: imagepath = args.input_image.name resultfilename = get_config_data('resultfilepath') + \ '_' + \ os.path.splitext(os.path.basename(imagepath))[0] \ + '.res' migrate_data.resultfilename = resultfilename migrate_tools.result_msg(msg='\n Running %s at %s\n' % ((os.path.basename(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])), time.ctime()), flags='w', result=True) else: raise OciMigrateException('Missing argument: input image path.') # # Import the 'format' modules and collect the format data supported_formats = migrate_tools.import_formats() if not bool(supported_formats): exit_with_msg(' *** ERROR *** No image format modules found') if migrate_data.verbose_flag: show_supported_formats_data(supported_formats) # # Check the utilities installed. util_list, missing_list = test_helpers() _logger.debug('%s' % util_list) if migrate_data.verbose_flag: show_utilities(util_list, missing_list) if missing_list: raise OciMigrateException('%s needs packages %s installed.\n' % (sys.argv[0], missing_list)) # # if qemu-img is used, the minimal version is 2 qemuversioninfo = qemu_img_version() if qemuversioninfo[1] < 2: raise OciMigrateException('Minimal version of qemu-img is 2, ' '%s found.' % qemuversioninfo[0]) else: _logger.debug('release data ok') # # Get the nameserver definition if system_tools.get_nameserver(): migrate_tools.result_msg(msg='nameserver %s identified.' % migrate_data.nameserver, result=False) _logger.debug('Nameserver identified as %s' % migrate_data.nameserver) else: migrate_tools.error_msg('Failed to identify nameserver, using %s, ' 'but might cause issues.' % migrate_data.nameserver) except Exception as e: _logger.error('*** ERROR *** %s\n' % str(e)) exit_with_msg(' *** ERROR *** %s\n' % str(e)) # # More on command line arguments. # # initialise output migrate_tools.result_msg(msg='Results are written to %s.' % migrate_data.resultfilename, result=True) migrate_tools.result_msg(msg='Input image: %s' % imagepath, result=True) # # Verify if readable. fn_magic = migrate_tools.get_magic_data(imagepath) if fn_magic is None: exit_with_msg('*** ERROR *** An error occured while trying to read ' 'magic number of File %s.' % imagepath) else: pause_msg('Image Magic Number: %s' % fn_magic) _logger.debug('Magic number %s successfully read' % fn_magic) # # Verify if image type is supported. _logger.debug('Magic number of %s is %s' % (imagepath, fn_magic)) if fn_magic not in supported_formats: exit_with_msg('*** ERROR *** Image type %s is not recognised.' % fn_magic) else: _logger.debug('Image type recognised.') # # Get the correct class. imageclazz = supported_formats[fn_magic] migrate_tools.result_msg(msg='Type of image %s identified as %s' % (imagepath, imageclazz['name']), result=True) pause_msg('Image type is %s' % imageclazz['name']) # # Locate the class and module imageclassdef = getattr(sys.modules['oci_utils.migrate.image_types.%s' % supported_formats[fn_magic]['name']], imageclazz['clazz']) image_object = imageclassdef(imagepath) # # Local volume groups. vgs_result = system_tools.exec_vgs_noheadings() migrate_data.local_volume_groups = \ vgs_result if bool(vgs_result) \ else [] _logger.debug('Workstation volume groups: %s' % migrate_data.local_volume_groups) # # Rename local volume groups if bool(migrate_data.local_volume_groups): rename_msg = '\n The workstation has logical volumes defined. To avoid ' \ 'duplicates, the \n logical volume groups will be temporary' \ ' renamed to a hexadecimal uuid.\n If you are sure the ' \ 'image to be uploaded does not contain logical volumes,\n' \ ' or there are no conflicting volume group names, '\ 'the rename can be skipped\n\n Keep the volume group names?' if not read_yn(rename_msg, waitenter=True, suppose_yes=migrate_data.yes_flag): if migrate_tools.verify_local_fstab(): fstab_msg = '\n The fstab file on this workstation seems to ' \ 'contain device references\n using /dev/mapper ' \ 'devices. The volume group names on this ' \ 'workstation\n will be renamed temporarily. ' \ '/dev/mapper devices referring to logical volumes\n' \ ' can create problems in this context. To avoid ' \ 'this situation\n exit now and modify the ' \ 'device specification to LABEL or UUID.\n\n Continue?' if not read_yn(fstab_msg, waitenter=True, suppose_yes=migrate_data.yes_flag): exit_with_msg('Exiting') _logger.debug('Rename local volume groups to avoid conflicts.') vgrename_res = system_tools.exec_rename_volume_groups( migrate_data.local_volume_groups, 'FORWARD') migrate_data.local_volume_group_rename = True else: _logger.debug('fstab file has no /dev/mapper devices.') else: _logger.debug('Not renaming the volume groups.') _ = system_tools.reset_vg_list(migrate_data.local_volume_groups) else: _logger.debug('No local volume groups, no conflicts.') # # Generic data collection try: imgres, imagedata = collect_image_data(image_object) if migrate_data.verbose_flag: migrate_tools.show_image_data(image_object) if imgres: _logger.debug('Image processing succeeded.') else: _logger.critical(' Image processing failed.', exc_info=False) # if imagedata: _logger.debug('%s passed.' % imagepath) else: _logger.critical(' %s failed.' % imagepath, exc_info=False) except Exception as e: _logger.critical(' %s failed: %s' % (imagepath, str(e))) exit_with_msg('*** ERROR *** Problem detected during investigation of ' 'the image %s: %s, exiting.' % (imagepath, str(e))) # # Restore volume group names. if migrate_data.local_volume_group_rename: _logger.debug('Restore local volume groups.') vgrename_res = system_tools.exec_rename_volume_groups( migrate_data.local_volume_groups, 'BACKWARD') else: _logger.debug('No local volume group names to restore.') # # passed prerequisites and changes? prereq_passed = True # # Image type specific prerequisites prereq_msg = '' sup, msg = image_object.type_specific_prereq_test() if sup: migrate_tools.result_msg(msg='%s' % msg, result=True) else: prereq_passed = False prereq_msg = msg # # Generic prerequisites verification try: gen_prereq, msg = image_object.generic_prereq_check() if gen_prereq: prereq_msg += '\n %s passed the generic preqrequisites.' % imagepath else: prereq_passed = False prereq_msg += msg # if imgres: prereq_msg += '\n\n %s data collection and processing succeeded.' \ % imagepath else: prereq_passed = False # if prereq_passed: migrate_tools.result_msg(msg=prereq_msg, result=True) if imagedata['boot_type'] == 'BIOS': migrate_tools.result_msg(msg='\n Boot type is BIOS, ' 'use launch_mode PARAVIRTUALIZED ' '(or EMULATED) at import.', result=True) elif imagedata['boot_type'] == 'UEFI': migrate_tools.result_msg(msg='\n Boot type is UEFI, ' 'use launch_mode NATIVE ' '(or EMULATED) at import.', result=True) else: raise OciMigrateException('Something wrong checking ' 'the boot type') else: prereq_msg += '\n\n %s processing failed, check the logfile ' \ 'and/or set environment variable _OCI_UTILS_DEBUG.' \ % imagepath raise OciMigrateException(prereq_msg) except Exception as e: exit_with_msg('*** ERROR *** %s' % str(e)) # # While prerequisite check did not hit a fatal issue, there might be # situations where upload should not proceed. if not migrate_data.migrate_preparation: exit_with_msg('*** ERROR *** Unable to proceed with uploading image ' 'to Oracle Cloud Infrastructure: %s' % migrate_data.migrate_non_upload_reason) else: migrate_tools.result_msg('Successfully verified and processed image %s ' 'and is ready for upload.' % imagepath) return 0