def uninstall_pkg(pkg_name): """ Uninstall rpm/deb package by name :param pkg_name: name of pkg to uninstall :type: String :return: Flag whether pkg uninstalled :rtype: bool """ log = logging.getLogger('package_manager.uninstall_pkg') if not is_pkg_installed(pkg_name): log.info(f'Package {pkg_name} was removed') return True cmd = _CMD_PATTERN["UNINSTALL"].get( get_os_name()).format(pkg_name=pkg_name) err, out = cmd_exec(cmd) if err == 0: log.info(f'Package {pkg_name} was removed') log.debug(out) return True log.error(out) log.error(f'Package "{pkg_name}" was not removed') return False
def _get_psnr(self, first_file, second_file, width, height): """ Get PSNR value :param first_file: Name of first file to compare :param second_file: Name of second file to compare :param width: Width of sequences pixels :param height: Height of sequences pixels :return: PSNR | False """ self.log.info('-' * 80) self.log.info('Get PSNR') metrics_calc_cmd = [ 'metrics_calc_lite', '-i1', '-i2', '-w', '-h', 'psnr', 'ssim', 'all' ] metrics_calc_cmd.insert( metrics_calc_cmd.index('-i1') + 1, str(first_file)) metrics_calc_cmd.insert( metrics_calc_cmd.index('-i2') + 1, str(second_file)) metrics_calc_cmd.insert(metrics_calc_cmd.index('-w') + 1, width) metrics_calc_cmd.insert(metrics_calc_cmd.index('-h') + 1, height) code, out = cmd_exec(metrics_calc_cmd, shell=False, log=self.log) if code: self.log.error(out) return False self.log.info(out) psnr = re.search('<avg_metric=PSNR>(.*)</avg_metric>', out).group(1) return psnr
def _check_md5(self): """ Compare reference md5sum with actual :return: Boolean """ self.log.info('-' * 80) self.log.info('Check md5 sum') self.log.info(self._ref_value) self.log.info(self._output_file) code, out = cmd_exec(['md5sum', self._output_file], shell=False, log=self.log) if code: self.log.error(out) return False self.log.info(out) md5sum, _ = out.split(' ') self.log.info(f'reference md5: {self._ref_value}') self.log.info(f'actual md5: {md5sum}') if self._ref_value != md5sum: return False return True
def run(self, options=None): """ Script runner :return: None | subprocess.CalledProcessError """ self.log.info('-' * 50) if self.callfunc: func, args, kwargs = self.callfunc self.log.info('function: %s', func) self.log.info('args: %s', args) self.log.info('kwargs: %s', kwargs) try: func(*args, **kwargs) except Exception as e: error_code = ErrorCode.CRITICAL.value self.log.error(e) return error_code if self.cmd: if isinstance(self.cmd, list): self.cmd = ' && '.join(self.cmd) env = os.environ.copy() if options: self.cmd = self.cmd.format_map(options) if options.get('ENV'): env.update(options['ENV']) if self.env: env.update(self.env) if self.work_dir: self.work_dir.mkdir(parents=True, exist_ok=True) error_code, out = cmd_exec(self.cmd, env=env, cwd=self.work_dir, log=self.log) if error_code: self._parse_logs(out) else: if self.verbose: self.log.info(out) self.log.info('completed') else: self.log.debug(out) self.log.debug('completed') return error_code
def _env_check(self): """ Check test environment :return: Boolean """ self.log.info('-' * 80) self.log.info('Check environment') code, _ = cmd_exec('lsmod | grep -q i915', log=self.log) if code: self.log.error("system does not load i915 module") return False self.log.info("i915 load successfully") code, vainfo = cmd_exec('vainfo', log=self.log) if code: self.log.info(vainfo) return False try: driver_version = re.search(f'.*Driver version: (.*)', vainfo).group(1).strip() except Exception: self.log.exception('Exception occurred:') return False self.log.info(f'Driver version: {driver_version}') if 'iHD' in driver_version: self.log.info( 'Current media driver is iHD and supports gen11+ and Whiskylake platform' ) elif 'i965' in driver_version: self.log.info( 'Current media driver is i965 and supports pre-gen11 platform') else: self.log.error('Unknown media driver') return False return True
def _execute_test_cmd(self): """ Run base command line of a test :return: Boolean """ self.log.info('-' * 80) self.log.info('Execute test command line') code, out = cmd_exec(self._cmd, shell=False, log=self.log) if code: self.log.error(out) return False self.log.info(out) return True
def _git_commit(self): self._log.info('Pushing changes to remote') push_change_commands = [f'git checkout -b {self._work_branch}', 'git add manifest.yml', f'git commit -m "{self._commit_message}"', f'git push origin HEAD:{self._work_branch}'] try: for command in push_change_commands: repo_path = self._tmp_dir / self._repo_name return_code, output = cmd_exec(command, cwd=repo_path, log=self._log) if return_code: self._log.error(output) return False except Exception as e: self._log.exception('Pushing was failed: %s', e) return False return True
def install_pkg(pkg_path): """ :param pkg_path: path to pkg to install :type: pathlib.Path :return: Flag whether pkg installed :rtype: bool """ configure_logger('package_manager.install_pkg') log = logging.getLogger() cmd = _CMD_PATTERN["INSTALL"].get(get_os_name()).format(pkg_path=pkg_path) err, out = cmd_exec(cmd) if err == 0: log.debug(out) return True log.info(out) return False
def install_pkg(pkg_path): """ :param pkg_path: path to pkg to install :type: pathlib.Path :return: Flag whether pkg installed :rtype: bool """ log = logging.getLogger('package_manager.install_pkg') cmd = _CMD_PATTERN["INSTALL"].get(get_os_name()).format(pkg_path=pkg_path) err, out = cmd_exec(f'{cmd} && sudo ldconfig') if err == 0: log.info(f'Package "{pkg_path}" was installed') log.debug(out) return True log.error(out) log.error(f'Package "{pkg_path}" was not installed') return False
def is_pkg_installed(pkg_name): """ Check whether pkg is installed :param pkg_name: pkg name :type: String :return: Flag whether pkg is installed :rtype: bool """ log = logging.getLogger('package_manager.is_pkg_installed') cmd = _CMD_PATTERN["CHECK_INSTALLED"].get( get_os_name()).format(pkg_name=pkg_name) err, out = cmd_exec(cmd) if err == 0: log.info(f'Package {pkg_name} was found') log.debug(out) return True log.info(out) return False
def _compare_files(self, first_file, second_file): """ Compare two files byte by byte :param first_file: Name of first file to compare :param second_file: Name of second file to compare :return: Boolean """ self.log.info('-' * 80) self.log.info('Compare files') code, out = cmd_exec( ['cmp', str(first_file), str(second_file)], shell=False, log=self.log) if code: self.log.warning('md5 checksum IS NOT SAME with ffmpeg sw decode') self.log.warning(out) return False self.log.info('md5 checksum IS SAME with ffmpeg sw decode') return True
def _strip_bins(self): """ Strip binaries and save debug information :return: Boolean """ self._log.info('-' * 80) self._log.info(f'Stripping binaries') system_os = platform.system() if system_os == 'Linux': bins_to_strip = [] binaries_with_error = [] executable_bin_filter = ['', '.so'] search_results = self._options['BUILD_DIR'].rglob('*') for path in search_results: if path.is_file(): if os.access( path, os.X_OK) and path.suffix in executable_bin_filter: bins_to_strip.append(path) for result in bins_to_strip: orig_file = str(result.absolute()) debug_file = str( (result.parent / f'{result.stem}.sym').absolute()) self._log.debug('-' * 80) self._log.debug(f'Stripping {orig_file}') strip_commands = OrderedDict([ ('copy_debug', ['objcopy', '--only-keep-debug', orig_file, debug_file]), ('strip', [ 'strip', '--strip-debug', '--strip-unneeded', '--remove-section=.comment', orig_file ]), ('add_debug_link', [ 'objcopy', f'--add-gnu-debuglink={debug_file}', orig_file ]), ('set_chmod', ['chmod', '-x', debug_file]) ]) check_binary_command = f'file {orig_file} | grep ELF' for command in strip_commands.values(): err, out = cmd_exec(command, shell=False, log=self._log, verbose=False) if err: # Not strip file if it is not binary return_code, _ = cmd_exec(check_binary_command, shell=True, log=self._log, verbose=False) if return_code: self._log.warning( f"File {orig_file} is not binary") break if orig_file not in binaries_with_error: binaries_with_error.append(orig_file) self._log.error(out) continue if binaries_with_error: self._log.error( 'Stripping for next binaries was failed. ' 'See full log for details:\n%s', '\n'.join(binaries_with_error)) return False elif system_os == 'Windows': pass else: self._log.error(f'Can not strip binaries on {system_os}') return False return True
def increase_build_number(local_repo_path, component, branch): """ Increase build number by 1 in remote repository, if it is the same for local and remote repositories This condition is needed to avoid increasing build number while rebuilds Function extracts product-configs repo in following layout to push change to remote repository ../tmp/product-configs ../origin_repo_path :param local_repo_path: path to local repository with "build_numbers.json" file :type local_repo_path: String | pathlib.Path :param component: Component name. Need for finding certain build number :type component: String :param branch: Name of branch. Need for finding certain build number :type branch: String """ log = logging.getLogger('build_number.increase_build_number') log.info(f'Increasing build number in {branch} branch for {component}') build_numbers_file = 'build_numbers.json' log.info(f'Get build number from local repository') current_build_number = get_build_number( repo_path=pathlib.Path(local_repo_path), component=component, branch=branch) if current_build_number == 0: log.error( f'Local build number must not be 0\n' f'Check that {pathlib.Path(local_repo_path) / build_numbers_file} contains appropriate {branch} branch for {component}' ) return False repo_name = pathlib.Path(local_repo_path).name temp_dir = (pathlib.Path(local_repo_path) / '..' / 'tmp').resolve() latest_version_repo_path = temp_dir / repo_name latest_build_number_path = latest_version_repo_path / build_numbers_file if temp_dir.exists(): log.info(f"Remove old repository in {temp_dir}") remove_directory(str(temp_dir)) temp_dir.mkdir(exist_ok=True) log.warning(f'Redefine {branch} to "master"') # TODO: use extract_repo from git_worker in one_ci_dev branch branch = 'master' extract_repo(root_repo_dir=temp_dir, repo_name=repo_name, branch=branch, commit_id='HEAD') log.info( f'Getting build number from HEAD of {branch} branch for repo in {latest_version_repo_path}' ) latest_git_build_number = get_build_number( repo_path=latest_version_repo_path, component=component, branch=branch) if current_build_number != latest_git_build_number: log.warning( f'Build numbers in remote ({latest_git_build_number}) and local ({current_build_number}) repositories are not equal\n' f'It maybe because this is rebuild of old build for which build number already has been increased\n' f'Stop operation') return False log.info('Increasing build number') if latest_build_number_path.exists(): try: log.info(f'\tChanging build numbers file') with latest_build_number_path.open('r+') as build_number_file: build_numbers = json.load(build_number_file) new_build_number = build_numbers[component][branch] + 1 build_numbers[component][branch] = new_build_number build_number_file.seek(0) build_number_file.write( json.dumps(build_numbers, indent=4, sort_keys=True)) build_number_file.truncate() log.info(f'\tPush changes') push_change_commands = [ 'git add -A', f'git commit -m "Increased build number of {branch} branch for {component} to {new_build_number}"', f'git push origin HEAD:{branch}' ] for command in push_change_commands: return_code, output = cmd_exec(command, cwd=latest_version_repo_path) if return_code: log.error(output) except Exception: log.exception('Exception occurred') return False else: log.error( f'Increasing build number failed, because {latest_build_number_path} does not exist' ) return False log.info(f'Build number was increased to {new_build_number}') return True
def _check_psnr(self): """ Check PSNR consistence :return: Boolean """ self.log.info('-' * 80) self.log.info('Check PSNR') psnr = None _output_file = pathlib.Path(self._output_file) if not _output_file.exists(): self.log.error(f'{_output_file} does not exist') self.log.error(f'reference psnr: {self._ref_value}') self.log.error(f'actual psnr: {psnr}') return False cmd_copy = copy(self._cmd) if self._feature == 'decode': self.log.info('Decode the input file with ffmpeg sw') sw_output_file = _output_file.parent / f'{_output_file.name}_sw.yuv' # remove -hwaccel idx = cmd_copy.index('-hwaccel') cmd_copy.pop(idx) cmd_copy.pop(idx) # remove -hwaccel_device idx = cmd_copy.index('-hwaccel_device') cmd_copy.pop(idx) cmd_copy.pop(idx) # update output file cmd_copy[-1] = str(sw_output_file) # execute ffmpeg sw decode code, out = cmd_exec(cmd_copy, shell=False, log=self.log) if code: self.log.error(out) return False self.log.info(out) resolution = re.search(rf'.*Stream #.*, (\d*x\d*).*', out).group(1).strip() self._width, self._height = resolution.split('x') # compare outputs if not self._compare_files(self._output_file, sw_output_file): psnr = self._get_psnr(sw_output_file, self._output_file, self._width, self._height) elif self._feature == 'encode': self.log.info('Encode the input file with ffmpeg sw') output_yuv = _output_file.parent / f'{_output_file.name}.yuv' # get -vframes vframes = cmd_copy[cmd_copy.index('-vframes') + 1] ffmpeg_cmd = [ 'ffmpeg', '-v', 'debug', '-i', self._output_file, '-pix_fmt', 'yuv420p', '-f', 'rawvideo', '-vsync', 'passthrough', '-vframes', vframes, '-y', str(output_yuv) ] code, out = cmd_exec(ffmpeg_cmd, shell=False, log=self.log) if code: self.log.error(out) return False self.log.info(out) psnr = self._get_psnr(self._input_file, output_yuv, self._width, self._height) elif self._feature == 'vp': self.log.info('Scale the input file with ffmpeg sw') if '-vf' in cmd_copy: vf_args = cmd_copy[cmd_copy.index('-vf') + 1] if 'scale_vaapi' in vf_args: sw_output_file = _output_file.parent / f'{_output_file.name}_sw.yuv' self._width, self._height = re.search( r'.*scale_vaapi=w=(\d+):h=(\d+).*', vf_args).groups() self.log.info( f'Scale: width = {self._width}; height = {self._height}' ) # remove -hwaccel idx = cmd_copy.index('-hwaccel') cmd_copy.pop(idx) cmd_copy.pop(idx) # remove -vaapi_device idx = cmd_copy.index('-vaapi_device') cmd_copy.pop(idx) cmd_copy.pop(idx) # update -vf cmd_copy[cmd_copy.index('-vf') + 1] = f'scale={self._width}:{self._height}' # update output file cmd_copy[-1] = str(sw_output_file) # scaling with ffmpeg sw code, out = cmd_exec(cmd_copy, shell=False, log=self.log) if code: self.log.error(out) return False self.log.info(out) # compare outputs if not self._compare_files(self._output_file, sw_output_file): psnr = self._get_psnr(sw_output_file, self._output_file, self._width, self._height) else: self.log.error(f'Feature {self._feature} is not supported') return False if psnr is None: return True if not psnr: return False self.log.info(f'reference psnr: {self._ref_value}') self.log.info(f'actual psnr: {psnr}') psnr_gap = 100 * (float(psnr) - float(self._ref_value)) / float( self._ref_value) psnr_gap = round(psnr_gap, 4) self.log.info(f'psnr gap: {psnr_gap}%') if psnr_gap < -5: return False return True
def run(self, options=None): """ Script runner :return: None | subprocess.CalledProcessError """ self.log.info('-' * 50) if self.callfunc: #TODO: Modify and restore ENV for callfunc too func, args, kwargs = self.callfunc self.log.info('function: %s', func) if args: self.log.info('args: %s', args) if kwargs: self.log.info('kwargs: %s', kwargs) try: return_value = func(*args, **kwargs) return_msg = f'The function returned: {return_value}' if isinstance(return_value, bool) and return_value == False: self.log.error(return_msg) error_code = ErrorCode.CRITICAL.value else: self.log.info(return_msg) error_code = ErrorCode.SUCCESS.value except Exception: error_code = ErrorCode.CRITICAL.value self.log.exception('Failed to call the function:') elif self.cmd: if isinstance(self.cmd, list): self.cmd = ' && '.join(self.cmd) env = os.environ.copy() if options: self.cmd = self.cmd.format_map(options) if options.get('ENV'): env.update(options['ENV']) if self.env: env.update(self.env) if self.work_dir: self.work_dir.mkdir(parents=True, exist_ok=True) error_code, out = cmd_exec(self.cmd, env=env, cwd=self.work_dir, log=self.log) if error_code: self._parse_logs(out) else: if self.verbose: self.log.info(out) self.log.info('completed') else: self.log.debug(out) self.log.debug('completed') else: error_code = ErrorCode.CRITICAL.value self.log.critical( f'The action "{self.name}" has not cmd or callfunc parameters') return error_code