def set_nameserver(): """ Setting temporary nameserver. Returns ------- bool: True on success, False otherwise. """ # global resolv_conf_path _logger.debug('__ Set nameserver.') # # rename eventual existing resolv.conf resolvpath = '/etc/resolv.conf' try: # # save current if os.path.isfile(resolvpath) or os.path.islink(resolvpath) or os.path.isdir(resolvpath): migrate_data.resolv_conf_path = exec_rename(resolvpath) if not bool(migrate_data.resolv_conf_path): _logger.debug('Failed to save current nameserver configuration.') else: _logger.error(' No %s found', resolvpath) # # write new with open(resolvpath, 'w') as f: f.writelines('nameserver %s\n' % migrate_data.nameserver) return True except Exception as e: error_msg('Failed to set nameserver: %s\n continuing but might cause issues installing cloud-init.' % str(e)) return False
def unmount_lvm2(vg): """ Remove logical volume data from system. Parameters ---------- vg: dict Volume group with list of logical volumes. Returns ------- bool: True on Success, exception otherwise. """ _logger.debug('__ Remove %s from system.', vg) try: # # make unavailable for vg_name in list(vg.keys()): vgchange_args = ['--activate', 'n', vg_name] vgchange_res = system_tools.exec_vgchange(vgchange_args) _logger.debug('vgchange: %s', vgchange_res) # # remove physical volume: clear cache, if necessary if system_tools.exec_pvscan(['--cache']): _logger.debug('pvscan clear succeeded') else: _logger.error(' pvscan failed') except Exception as e: _logger.error(' Failed to release lvms %s: %s', vg, str(e)) error_msg('Failed to release lvms %s: %s' % (vg, str(e)))
def restore_nameserver(): """ Restore nameserver configuration. Returns ------- bool: True on success, False otherwise. """ _logger.debug('__ Restore nameserver data.') # global resolv_conf_path resolvpath = '/etc/resolv.conf' try: # # save used one if os.path.isfile(resolvpath): if not bool(exec_rename(resolvpath, resolvpath + '_temp_' + migrate_data.current_time)): _logger.debug('Failed to rename %s to %s, no harm done.', resolvpath, resolvpath + '_temp_' + migrate_data.current_time) else: _logger.debug('No %s found.', resolvpath) # # restore original one if os.path.isfile(migrate_data.resolv_conf_path): if bool(exec_rename(migrate_data.resolv_conf_path, resolvpath)): _logger.debug('Successfully restored %s', resolvpath) else: _logger.debug('Failed to restore %s.', resolvpath) raise OciMigrateException('Failed to restore nameserver config.') else: _logger.debug('No %s found.', migrate_data.resolv_conf_path) return True except Exception as e: error_msg('Continuing but might cause issues installing cloud-init: %s' % str(e)) return False
def a20_install_extra_pkgs(self): """ Install required and useful packages for OL-type installs, read the list of packages from oci-migrate-conf.yaml, ol_os_packages_to_install. Returns ------- bool: True on success, False otherwise. """ def get_ol_package_list(): """ Retrieve the list of packages to install from oci-migrate-config file. Returns ------- list: list of package names to install. """ _logger.debug('Collection list of packages.') try: pkg_list = get_config_data('ol_os_packages_to_install') if not bool(pkg_list): _logger.debug('Package list is empty.') return False _logger.debug('Package list: %s', pkg_list) return pkg_list except Exception as e: _logger.warning('Failed to find a list of packages: %s', str(e)) return False _logger.debug('__ Installing extra packages.') # # collect packages to install packages = get_ol_package_list() if not bool(packages): _logger.debug('No extra packages to install.') return True # # try: # # set current nameserver config. if system_tools.set_nameserver(): _logger.debug('Updating nameserver info succeeded.') else: _logger.error(' Failed to update nameserver info.') # # get package manipulation tool. pkg_mgr = self._exec_yum \ if self.package_tool['pkg_mgr'] == 'yum' else self._exec_dnf # # install packages for pkg in packages: # # verify if the package is available, the correct channel enabled. rpmlist = pkg_mgr(self.package_tool['package_available'] + [pkg]) pkg_present = False for lline in rpmlist.splitlines(): _logger.debug('%s', lline) if pkg in lline: _logger.debug('The rpm %s is available.', pkg) pkg_present = True break if not pkg_present: _logger.error(' The rpm %s is missing.', pkg) migrate_data.migrate_preparation = False migrate_data.migrate_non_upload_reason += \ '\n The rpm package %s is missing from ' \ 'the yum repository.' % pkg return False installoutput = pkg_mgr(self.package_tool['package_install'] + [pkg]) _logger.debug('Successfully installed pkg %s:\n%s', pkg, installoutput) result_msg(msg='Installed %s.' % pkg, result=False) pause_msg(msg='Installed %s here, or not.' % pkg, pause_flag='_OCI_CHROOT') # # restore nameserver data if system_tools.restore_nameserver(): _logger.debug('Restoring nameserver info succeeded.') else: _logger.error(' Failed to restore nameserver info.') except Exception as e: errmsg = 'Failed to install one or more packages of ' \ '%s:\n%s' % (packages, str(e)) _logger.critical(' %s', errmsg) error_msg('%s' % errmsg) migrate_data.migrate_preparation = False migrate_data.migrate_non_upload_reason += '\n %s' % errmsg return False return True
def reconfigure_ifcfg_config(rootdir): """ Modify the network configuration in the image file to prevent conflicts during import. This is only for ol-type linux. Parameters ---------- rootdir: str Full path of image root dir as loopback mounted. Returns ------- list: list of nic. dict: the interfaces configuration. """ # # Rename the config files _logger.debug('__ The network ifcfg configuration.') ifcfg_list = list() ifcfg_data = dict() ifrootdir = rootdir + get_config_data('default_ifcfg') if os.path.isdir(ifrootdir): for cfgfile in glob(ifrootdir + '/ifcfg-*'): _logger.debug('Checking configfile: %s', cfgfile) try: with open(cfgfile, 'r') as f: # nl = filter(None, [x[:x.find('#')] for x in f]) nl = [_f for _f in [x[:x.find('#')] for x in f] if _f] ifcfg = dict(dl.replace('"', '').split('=') for dl in nl) if 'DEVICE' in ifcfg: devname = ifcfg['DEVICE'] else: _logger.debug('Missing device name in %s', cfgfile) devname = cfgfile.split('/')[-1] ifcfg_list.append(devname) ifcfg_data[devname] = ifcfg _logger.debug('Network interface: %s', devname) except Exception as e: _logger.error( ' Problem reading network configuration file %s: %s', cfgfile, str(e)) else: _logger.debug('No ifcfg network configuration.') # # backup for fn in glob(ifrootdir + '/ifcfg-*'): if 'ifcfg-lo' not in fn: fn_bck = system_tools.exec_rename(fn) if bool(fn_bck): _logger.debug( 'Network config file %s successfully renamed to %s', fn, fn_bck) else: _logger.debug( 'Failed to backup network configuration file %s to %s.', fn, fn_bck) # error_msg('Failed to backup network configuration ' # 'file %s to %s.' % (fn, fn_bck)) # raise OciMigrateException('Failed to rename network config ' # 'file %s to %s' % (fn, fn_bck)) else: _logger.debug('ifcfg-lo found.') # # Generate new default network configuration. if len(ifcfg_list) > 0: nic0 = sorted(ifcfg_list)[0] dhcpniccfg = ifrootdir + '/ifcfg-%s' % nic0 _logger.debug('Replacing network config file %s', dhcpniccfg) try: with open(dhcpniccfg, 'w') as f: f.writelines( ln.replace('_XXXX_', nic0) + '\n' for ln in get_config_data('default_ifcfg_config')) result_msg(msg='Replaced ifcfg network configuration.', result=False) except Exception as e: _logger.error(' Failed to write %s/ifcfg-eth0', ifrootdir) error_msg('Failed to write %s: %s' % (dhcpniccfg, str(e))) raise OciMigrateException('Failed to write %s:' % dhcpniccfg) from e else: _logger.debug('No ifcfg definitions found.') return ifcfg_list, ifcfg_data
def reconfigure_networkmanager(rootdir): """ Replace the networkmanager configuration with Oracle Cloud Infrastructure compatible version. Parameters ---------- rootdir: str Full path of image root dir as loopback mounted. Returns ------- list: list of nic. dict: the network manager system-connections configurations """ _logger.debug('__ The NetworkManager configuration.') netwmg_data = dict() netwmg_nics = list() network_config_dir = rootdir + get_config_data('default_nwconnections') _logger.debug('Network manager dir: %s', network_config_dir) nw_mgr_cfg = rootdir + get_config_data('default_nwmconfig') _logger.debug('Network manager conf: %s', nw_mgr_cfg) # # backup try: # # copy if os.path.isfile(nw_mgr_cfg): bck_nw_mgr_cfg = system_tools.exec_rename(nw_mgr_cfg) if bool(bck_nw_mgr_cfg): _logger.debug('Copied %s to %s', nw_mgr_cfg, bck_nw_mgr_cfg) else: _logger.warning( 'Failed to backup network manager configuration.') else: _logger.debug('No %s found.', nw_mgr_cfg) # if os.path.isdir(network_config_dir): bck_network_config_dir = system_tools.backup_dir( network_config_dir) _logger.debug('Copied %s to %s', network_config_dir, bck_network_config_dir) else: _logger.debug('%s not found.', network_config_dir) except Exception as e: error_msg('Failed to backup the networkmanager configuration: %s' % str(e)) # # if os.path.isdir(network_config_dir): _logger.debug('NetworkManager/%s directory exists.', network_config_dir) # # contains nwm keyfiles? nwm_files = glob(network_config_dir + '/*') if len(nwm_files) > 0: system_tools.exec_rmdir(network_config_dir) system_tools.exec_mkdir(network_config_dir) _logger.debug('%s emptied.', network_config_dir) else: _logger.debug('No network manager keyfiles found.') # # update networkmanager configuration # TODO: write config file with configparser nwm_config_data = get_config_data('default_nwm_conf_file') with open(nw_mgr_cfg, 'w') as nwmf: nwmf.write('\n'.join(str(x) for x in nwm_config_data)) result_msg(msg='Networkmanager configuration updated.', result=False) else: _logger.debug(msg=' No NetworkManager configuration present.') return netwmg_nics, netwmg_data
def reconfigure_netplan(rootdir): """ Parse the yaml netplan files and look for network interface names. Parameters ---------- rootdir: str Full path of image root dir as loopback mounted. Returns ------- list: list of nic. dict: the netplan network configurations. """ _logger.debug('__ The netplan configuration.') netplan_data = dict() netplan_nics = list() root_path = rootdir + get_config_data('default_netplan') if os.path.isdir(root_path): _logger.debug('netplan directory exists.') # # contains yaml files? yaml_files = glob(root_path + '/*.yaml') if len(yaml_files) > 0: for yf in sorted(yaml_files): try: with open(yf, 'r') as yfd: yaml_data = yaml.safe_load(yfd) netplan_data[yf] = yaml_data _logger.debug('netplan: %s', yaml_data) except Exception as e: _logger.error(' Failed to parse %s: %s', yf, str(e)) error_msg('Failed to parse %s: %s' % (yf, str(e))) break # if 'network' in yaml_data: if 'ethernets' in yaml_data['network']: for k, _ in sorted( yaml_data['network']['ethernets'].items()): netplan_nics.append(k) else: _logger.debug('ethernets key missing.') else: _logger.debug('network key missing.') if len(netplan_nics) == 0: _logger.debug('No netplan definitions found in %s', root_path) else: nicname = sorted(netplan_nics)[0] # # rename and recreate try: # # backup if not bool(system_tools.exec_rename(root_path)): _logger.warning('Failed to backup %s.', root_path) # # recreate dir os.mkdir(root_path) mode755 = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH | \ stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | \ stat.S_IROTH os.chmod(root_path, mode755) _logger.debug((' Recreated %s', root_path)) # # recreate netplan config # # GT commenting out this one to avoid conflicts, cloud-init recreates it from scratch. # # netplan_config = get_config_data('default_netplan_config') # netplan_config['network']['ethernets'][nicname] \ # = netplan_config['network']['ethernets'].pop('_XXXX_') # with open(root_path + '/' # + get_config_data('default_netplan_file'), 'w') \ # as yf: # yaml.safe_dump(netplan_config, yf, default_flow_style=False) # result_msg(msg='Netplan network configuration ' # 'files replaced.', result=True) except Exception as e: error_msg( 'Failed to write new netplan configuration file %s: %s' % (get_config_data('default_netplan_file'), str(e))) raise OciMigrateException( 'Failed to write new netplan configuration file %s' % (get_config_data('default_netplan_file'))) from e else: _logger.debug(' No netplan yaml config files found.') else: _logger.debug('No netplan configuration found.') return netplan_nics, netplan_data
def a20_install_extra_pkgs(self): """ Install required and useful packages for OL-type installs, read the list of packages from oci-migrate-conf.yaml, ol_os_packages_to_install. Returns ------- bool: True on success, False otherwise. """ def get_ubuntu_package_list(): """ Retrieve the list of packages to install from oci-migrate-config file. Returns ------- list: list of package names to install. """ _logger.debug('Collection list of packages.') try: pkg_list = get_config_data('ubuntu_os_packages_to_install_apt') if not bool(pkg_list): _logger.debug('apt package list is empty.') return False _logger.debug('Package list: %s', pkg_list) return pkg_list except Exception as e: _logger.warning('Failed to find a list of packages: %s', str(e)) return False _logger.debug('Installing extra packages.') packages = get_ubuntu_package_list() if not bool(packages): _logger.debug('No extra packages to install.') return True try: # # set current nameserver config. if system_tools.set_nameserver(): _logger.debug('Updating nameserver info succeeded.') else: _logger.error(' Failed to update nameserver info.') # # update package list update_output = self._exec_apt(['update']) _logger.debug('Successfully updated package list.') # # install packages for pkg in packages: # # verify if the package is available. pkg_present = False deblist = self._exec_apt(['list', pkg]) for lline in deblist.splitlines(): _logger.debug('%s', lline) if pkg in lline: _logger.debug('The deb package %s is available.', pkg) pkg_present = True break if not pkg_present: _logger.debug('The deb package %s is missing.', pkg) migrate_data.migrate_preparation = False migrate_data.migrate_non_upload_reason +=\ '\n The deb package %s is missing from ' \ 'the repository.' % pkg return False installoutput = self._exec_apt(['install', '-y', pkg]) _logger.debug('Successfully installed %s:\n%s', pkg, installoutput) pause_msg(msg='Installed %s here, or not.' % pkg, pause_flag='_OCI_CHROOT') if system_tools.restore_nameserver(): _logger.debug('Restoring nameserver info succeeded.') else: _logger.error(' Failed to restore nameserver info.') except Exception as e: _logger.critical( ' Failed to install one or more packages of %s:\n%s', packages, str(e)) error_msg('Failed to install one or more packages of %s:\n%s' % (packages, str(e))) migrate_data.migrate_non_upload_reason += \ '\n Failed to install on or more packages ' \ 'of %s: %s' % (packages, str(e)) return False return True
def main(): """ Main Returns ------- int 0 on success, 1 otherwise. """ # # set locale lc_all_to_set = get_config_data('lc_all') os.environ['LC_ALL'] = "%s" % lc_all_to_set _logger.debug('Locale set to %s', lc_all_to_set) # # python version python_version = sys.version_info[0] _logger.debug('Python version is %s', python_version) # # parse the commandline args = parse_args() # # Operator needs to be root. if system_tools.is_root(): _logger.debug('User is root.') else: exit_with_msg(' *** ERROR *** You must run this program with root ' 'privileges') # # Verbose mode is False by default migrate_data.verbose_flag = args.verbose_flag _logger.debug('Verbose level set to %s', migrate_data.verbose_flag) # # Yes flag migrate_data.yes_flag = args.yes_flag _logger.debug('Answer to yes/no questions supposed to be yes always.') # # collect and save configuration data migrate_data.oci_image_migrate_config = get_config_data('*') # try: # # input image if args.input_image: image_path = args.input_image.name result_filename = get_config_data('result_filepath') \ + '_' \ + os.path.splitext(os.path.basename(image_path))[0] \ + '.res' migrate_data.result_filename = result_filename result_msg(msg='\n Running %s at %s\n' % ((os.path.basename(sys.argv[0]) + ' ' + ' '.join(sys.argv[1:])), time.ctime()), flags='w', result=True) else: raise OciMigrateException('Missing argument: input image path.') # # Import the 'format' modules and collect the format data supported_formats = migrate_tools.import_formats() if not bool(supported_formats): exit_with_msg(' *** ERROR *** No image format modules found') if migrate_data.verbose_flag: show_supported_formats_data(supported_formats) # # Check the utilities installed. util_list, missing_list = test_helpers() _logger.debug('%s', util_list) if migrate_data.verbose_flag: show_utilities(util_list, missing_list) if missing_list: raise OciMigrateException('%s needs packages %s installed.\n' % (sys.argv[0], missing_list)) # # if qemu-img is used, the minimal version is 2 qemu_version_info = qemu_img_version() if qemu_version_info[1] < 2: raise OciMigrateException('Minimal version of qemu-img is 2, ' '%s found.' % qemu_version_info[0]) _logger.debug('release data ok') # # Get the nameserver definition if system_tools.get_nameserver(): result_msg(msg='nameserver %s identified.' % migrate_data.nameserver, result=False) _logger.debug('Nameserver identified as %s', migrate_data.nameserver) else: error_msg( 'Failed to identify nameserver, using %s, but might cause issues.' % migrate_data.nameserver) except Exception as e: _logger.error('*** ERROR *** %s\n', str(e)) exit_with_msg(' *** ERROR *** %s\n' % str(e)) # # More on command line arguments. # # initialise output result_msg(msg='Results are written to %s.' % migrate_data.result_filename, result=True) result_msg(msg='Input image: %s' % image_path, result=True) # # Verify if readable. fn_magic = migrate_tools.get_magic_data(image_path) if fn_magic is None: exit_with_msg('*** ERROR *** An error occurred while trying to read ' 'magic number of File %s.' % image_path) else: pause_msg('Image Magic Number: %s' % fn_magic) _logger.debug('Magic number %s successfully read', fn_magic) # # Verify if image type is supported. _logger.debug('Magic number of %s is %s', image_path, fn_magic) if fn_magic not in supported_formats: exit_with_msg('*** ERROR *** Image type %s is not recognised.' % fn_magic) else: _logger.debug('Image type recognised.') # # Get the correct class. image_clazz = supported_formats[fn_magic] result_msg(msg='Type of image %s identified as %s' % (image_path, image_clazz['name']), result=True) pause_msg('Image type is %s' % image_clazz['name']) # # Locate the class and module image_class_def = getattr( sys.modules['oci_utils.migrate.image_types.%s' % supported_formats[fn_magic]['name']], image_clazz['clazz']) image_object = image_class_def(image_path) # # Local volume groups. vgs_result = system_tools.exec_vgs_noheadings() migrate_data.local_volume_groups = vgs_result if bool(vgs_result) else [] _logger.debug('Workstation volume groups: %s', migrate_data.local_volume_groups) # # Rename local volume groups if bool(migrate_data.local_volume_groups): rename_msg = '\n The workstation has logical volumes defined. To avoid ' \ 'duplicates, the \n logical volume groups will be temporary' \ ' renamed to a hexadecimal uuid.\n If you are sure the ' \ 'image to be uploaded does not contain logical volumes,\n' \ ' or there are no conflicting volume group names, '\ 'the rename can be skipped\n\n Keep the volume group names?' if not read_yn( rename_msg, waitenter=True, suppose_yes=migrate_data.yes_flag): if migrate_tools.verify_local_fstab(): fstab_msg = '\n The fstab file on this workstation seems to ' \ 'contain device references\n using /dev/mapper ' \ 'devices. The volume group names on this ' \ 'workstation\n will be renamed temporarily. ' \ '/dev/mapper devices referring to logical volumes\n' \ ' can create problems in this context. To avoid ' \ 'this situation\n exit now and modify the ' \ 'device specification to LABEL or UUID.\n\n Continue?' if not read_yn(fstab_msg, waitenter=True, suppose_yes=migrate_data.yes_flag): exit_with_msg('Exiting.') _logger.debug('Rename local volume groups to avoid conflicts.') vgrename_res = system_tools.exec_rename_volume_groups( migrate_data.local_volume_groups, 'FORWARD') if not vgrename_res: _logger.warning('Failed to rename local volume groups.') if not read_yn( '\n Failed to rename the local volume groups. ' 'Continue on your own responsibility?', waitenter=True, suppose_yes=migrate_data.yes_flag): exit_with_msg('Exiting.') migrate_data.local_volume_group_rename = True else: _logger.debug('fstab file has no /dev/mapper devices.') else: _logger.debug('Not renaming the volume groups.') _ = system_tools.reset_vg_list(migrate_data.local_volume_groups) else: _logger.debug('No local volume groups, no conflicts.') # # Generic data collection try: imgres, imagedata = collect_image_data(image_object) if migrate_data.verbose_flag: migrate_tools.show_image_data(image_object) if imgres: _logger.debug('Image processing succeeded.') else: _logger.critical(' Image processing failed.', exc_info=False) # if imagedata: _logger.debug('%s passed.', image_path) else: _logger.critical(' %s failed.', image_path, exc_info=False) except Exception as e: _logger.critical(' %s failed: %s', image_path, str(e)) exit_with_msg('*** ERROR *** Problem detected during investigation of ' 'the image %s: %s, exiting.' % (image_path, str(e))) # # Restore volume group names. if migrate_data.local_volume_group_rename: _logger.debug('Restore local volume groups.') vgrename_res = system_tools.exec_rename_volume_groups( migrate_data.local_volume_groups, 'BACKWARD') if not vgrename_res: _logger.warning('Failed to restore local volume group names.') else: _logger.debug('No local volume group names to restore.') # # passed prerequisites and changes? prereq_passed = True # # Image type specific prerequisites prereq_msg = '' sup, msg = image_object.type_specific_prereq_test() if sup: result_msg(msg='%s' % msg, result=True) else: prereq_passed = False prereq_msg = msg # # Generic prerequisites verification try: gen_prereq, msg = image_object.generic_prereq_check() if gen_prereq: prereq_msg += '\n %s passed the generic preqrequisites.' % image_path else: prereq_passed = False prereq_msg += msg # if imgres: prereq_msg += '\n\n %s data collection and processing succeeded.' \ % image_path else: prereq_passed = False # if prereq_passed: result_msg(msg=prereq_msg, result=True) if imagedata['boot_type'] == 'BIOS': result_msg( msg= '\n Boot type is BIOS, use launch_mode PARAVIRTUALIZED (or EMULATED) at import.', result=True) elif imagedata['boot_type'] == 'UEFI': result_msg( msg= '\n Boot type is UEFI, use launch_mode NATIVE (or EMULATED) at import.', result=True) else: raise OciMigrateException('Something wrong checking ' 'the boot type') else: prereq_msg += '\n\n %s processing failed, check the logfile ' \ 'and/or set environment variable _OCI_UTILS_DEBUG.' \ % image_path raise OciMigrateException(prereq_msg) except Exception as e: exit_with_msg('*** ERROR *** %s' % str(e)) # # While prerequisite check did not hit a fatal issue, there might be # situations where upload should not proceed. if not migrate_data.migrate_preparation: exit_with_msg('*** ERROR *** Unable to proceed with uploading image ' 'to Oracle Cloud Infrastructure: %s' % migrate_data.migrate_non_upload_reason) else: result_msg( 'Successfully verified and processed image %s and is ready for upload.' % image_path) return 0