def uninstall_pkg(pkg_name):
"""
Uninstall rpm/deb package by name
:param pkg_name: name of pkg to uninstall
:type: String
:return: Flag whether pkg uninstalled
:rtype: bool
"""
log = logging.getLogger('package_manager.uninstall_pkg')
if not is_pkg_installed(pkg_name):
log.info(f'Package {pkg_name} was removed')
return True
cmd = _CMD_PATTERN["UNINSTALL"].get(
get_os_name()).format(pkg_name=pkg_name)
err, out = cmd_exec(cmd)
if err == 0:
log.info(f'Package {pkg_name} was removed')
log.debug(out)
return True
log.error(out)
log.error(f'Package "{pkg_name}" was not removed')
return False
def _get_psnr(self, first_file, second_file, width, height):
"""
Get PSNR value
:param first_file: Name of first file to compare
:param second_file: Name of second file to compare
:param width: Width of sequences pixels
:param height: Height of sequences pixels
:return: PSNR | False
"""
self.log.info('-' * 80)
self.log.info('Get PSNR')
metrics_calc_cmd = [
'metrics_calc_lite', '-i1', '-i2', '-w', '-h', 'psnr', 'ssim',
'all'
]
metrics_calc_cmd.insert(
metrics_calc_cmd.index('-i1') + 1, str(first_file))
metrics_calc_cmd.insert(
metrics_calc_cmd.index('-i2') + 1, str(second_file))
metrics_calc_cmd.insert(metrics_calc_cmd.index('-w') + 1, width)
metrics_calc_cmd.insert(metrics_calc_cmd.index('-h') + 1, height)
code, out = cmd_exec(metrics_calc_cmd, shell=False, log=self.log)
if code:
self.log.error(out)
return False
self.log.info(out)
psnr = re.search('<avg_metric=PSNR>(.*)</avg_metric>', out).group(1)
return psnr
def _check_md5(self):
"""
Compare reference md5sum with actual
:return: Boolean
"""
self.log.info('-' * 80)
self.log.info('Check md5 sum')
self.log.info(self._ref_value)
self.log.info(self._output_file)
code, out = cmd_exec(['md5sum', self._output_file],
shell=False,
log=self.log)
if code:
self.log.error(out)
return False
self.log.info(out)
md5sum, _ = out.split(' ')
self.log.info(f'reference md5: {self._ref_value}')
self.log.info(f'actual md5: {md5sum}')
if self._ref_value != md5sum:
return False
return True
def run(self, options=None):
"""
Script runner
:return: None | subprocess.CalledProcessError
"""
self.log.info('-' * 50)
if self.callfunc:
func, args, kwargs = self.callfunc
self.log.info('function: %s', func)
self.log.info('args: %s', args)
self.log.info('kwargs: %s', kwargs)
try:
func(*args, **kwargs)
except Exception as e:
error_code = ErrorCode.CRITICAL.value
self.log.error(e)
return error_code
if self.cmd:
if isinstance(self.cmd, list):
self.cmd = ' && '.join(self.cmd)
env = os.environ.copy()
if options:
self.cmd = self.cmd.format_map(options)
if options.get('ENV'):
env.update(options['ENV'])
if self.env:
env.update(self.env)
if self.work_dir:
self.work_dir.mkdir(parents=True, exist_ok=True)
error_code, out = cmd_exec(self.cmd,
env=env,
cwd=self.work_dir,
log=self.log)
if error_code:
self._parse_logs(out)
else:
if self.verbose:
self.log.info(out)
self.log.info('completed')
else:
self.log.debug(out)
self.log.debug('completed')
return error_code
def _env_check(self):
"""
Check test environment
:return: Boolean
"""
self.log.info('-' * 80)
self.log.info('Check environment')
code, _ = cmd_exec('lsmod | grep -q i915', log=self.log)
if code:
self.log.error("system does not load i915 module")
return False
self.log.info("i915 load successfully")
code, vainfo = cmd_exec('vainfo', log=self.log)
if code:
self.log.info(vainfo)
return False
try:
driver_version = re.search(f'.*Driver version: (.*)',
vainfo).group(1).strip()
except Exception:
self.log.exception('Exception occurred:')
return False
self.log.info(f'Driver version: {driver_version}')
if 'iHD' in driver_version:
self.log.info(
'Current media driver is iHD and supports gen11+ and Whiskylake platform'
)
elif 'i965' in driver_version:
self.log.info(
'Current media driver is i965 and supports pre-gen11 platform')
else:
self.log.error('Unknown media driver')
return False
return True
def _execute_test_cmd(self):
"""
Run base command line of a test
:return: Boolean
"""
self.log.info('-' * 80)
self.log.info('Execute test command line')
code, out = cmd_exec(self._cmd, shell=False, log=self.log)
if code:
self.log.error(out)
return False
self.log.info(out)
return True
def _git_commit(self):
self._log.info('Pushing changes to remote')
push_change_commands = [f'git checkout -b {self._work_branch}', 'git add manifest.yml',
f'git commit -m "{self._commit_message}"',
f'git push origin HEAD:{self._work_branch}']
try:
for command in push_change_commands:
repo_path = self._tmp_dir / self._repo_name
return_code, output = cmd_exec(command, cwd=repo_path, log=self._log)
if return_code:
self._log.error(output)
return False
except Exception as e:
self._log.exception('Pushing was failed: %s', e)
return False
return True
def install_pkg(pkg_path):
"""
:param pkg_path: path to pkg to install
:type: pathlib.Path
:return: Flag whether pkg installed
:rtype: bool
"""
configure_logger('package_manager.install_pkg')
log = logging.getLogger()
cmd = _CMD_PATTERN["INSTALL"].get(get_os_name()).format(pkg_path=pkg_path)
err, out = cmd_exec(cmd)
if err == 0:
log.debug(out)
return True
log.info(out)
return False
def install_pkg(pkg_path):
"""
:param pkg_path: path to pkg to install
:type: pathlib.Path
:return: Flag whether pkg installed
:rtype: bool
"""
log = logging.getLogger('package_manager.install_pkg')
cmd = _CMD_PATTERN["INSTALL"].get(get_os_name()).format(pkg_path=pkg_path)
err, out = cmd_exec(f'{cmd} && sudo ldconfig')
if err == 0:
log.info(f'Package "{pkg_path}" was installed')
log.debug(out)
return True
log.error(out)
log.error(f'Package "{pkg_path}" was not installed')
return False
def is_pkg_installed(pkg_name):
"""
Check whether pkg is installed
:param pkg_name: pkg name
:type: String
:return: Flag whether pkg is installed
:rtype: bool
"""
log = logging.getLogger('package_manager.is_pkg_installed')
cmd = _CMD_PATTERN["CHECK_INSTALLED"].get(
get_os_name()).format(pkg_name=pkg_name)
err, out = cmd_exec(cmd)
if err == 0:
log.info(f'Package {pkg_name} was found')
log.debug(out)
return True
log.info(out)
return False
def _compare_files(self, first_file, second_file):
"""
Compare two files byte by byte
:param first_file: Name of first file to compare
:param second_file: Name of second file to compare
:return: Boolean
"""
self.log.info('-' * 80)
self.log.info('Compare files')
code, out = cmd_exec(
['cmp', str(first_file), str(second_file)],
shell=False,
log=self.log)
if code:
self.log.warning('md5 checksum IS NOT SAME with ffmpeg sw decode')
self.log.warning(out)
return False
self.log.info('md5 checksum IS SAME with ffmpeg sw decode')
return True
def _strip_bins(self):
"""
Strip binaries and save debug information
:return: Boolean
"""
self._log.info('-' * 80)
self._log.info(f'Stripping binaries')
system_os = platform.system()
if system_os == 'Linux':
bins_to_strip = []
binaries_with_error = []
executable_bin_filter = ['', '.so']
search_results = self._options['BUILD_DIR'].rglob('*')
for path in search_results:
if path.is_file():
if os.access(
path,
os.X_OK) and path.suffix in executable_bin_filter:
bins_to_strip.append(path)
for result in bins_to_strip:
orig_file = str(result.absolute())
debug_file = str(
(result.parent / f'{result.stem}.sym').absolute())
self._log.debug('-' * 80)
self._log.debug(f'Stripping {orig_file}')
strip_commands = OrderedDict([
('copy_debug',
['objcopy', '--only-keep-debug', orig_file, debug_file]),
('strip', [
'strip', '--strip-debug', '--strip-unneeded',
'--remove-section=.comment', orig_file
]),
('add_debug_link', [
'objcopy', f'--add-gnu-debuglink={debug_file}',
orig_file
]), ('set_chmod', ['chmod', '-x', debug_file])
])
check_binary_command = f'file {orig_file} | grep ELF'
for command in strip_commands.values():
err, out = cmd_exec(command,
shell=False,
log=self._log,
verbose=False)
if err:
# Not strip file if it is not binary
return_code, _ = cmd_exec(check_binary_command,
shell=True,
log=self._log,
verbose=False)
if return_code:
self._log.warning(
f"File {orig_file} is not binary")
break
if orig_file not in binaries_with_error:
binaries_with_error.append(orig_file)
self._log.error(out)
continue
if binaries_with_error:
self._log.error(
'Stripping for next binaries was failed. '
'See full log for details:\n%s',
'\n'.join(binaries_with_error))
return False
elif system_os == 'Windows':
pass
else:
self._log.error(f'Can not strip binaries on {system_os}')
return False
return True
def increase_build_number(local_repo_path, component, branch):
"""
Increase build number by 1 in remote repository, if it is the same for local and remote repositories
This condition is needed to avoid increasing build number while rebuilds
Function extracts product-configs repo in following layout to push change to remote repository
../tmp/product-configs
../origin_repo_path
:param local_repo_path: path to local repository with "build_numbers.json" file
:type local_repo_path: String | pathlib.Path
:param component: Component name. Need for finding certain build number
:type component: String
:param branch: Name of branch. Need for finding certain build number
:type branch: String
"""
log = logging.getLogger('build_number.increase_build_number')
log.info(f'Increasing build number in {branch} branch for {component}')
build_numbers_file = 'build_numbers.json'
log.info(f'Get build number from local repository')
current_build_number = get_build_number(
repo_path=pathlib.Path(local_repo_path),
component=component,
branch=branch)
if current_build_number == 0:
log.error(
f'Local build number must not be 0\n'
f'Check that {pathlib.Path(local_repo_path) / build_numbers_file} contains appropriate {branch} branch for {component}'
)
return False
repo_name = pathlib.Path(local_repo_path).name
temp_dir = (pathlib.Path(local_repo_path) / '..' / 'tmp').resolve()
latest_version_repo_path = temp_dir / repo_name
latest_build_number_path = latest_version_repo_path / build_numbers_file
if temp_dir.exists():
log.info(f"Remove old repository in {temp_dir}")
remove_directory(str(temp_dir))
temp_dir.mkdir(exist_ok=True)
log.warning(f'Redefine {branch} to "master"')
# TODO: use extract_repo from git_worker in one_ci_dev branch
branch = 'master'
extract_repo(root_repo_dir=temp_dir,
repo_name=repo_name,
branch=branch,
commit_id='HEAD')
log.info(
f'Getting build number from HEAD of {branch} branch for repo in {latest_version_repo_path}'
)
latest_git_build_number = get_build_number(
repo_path=latest_version_repo_path, component=component, branch=branch)
if current_build_number != latest_git_build_number:
log.warning(
f'Build numbers in remote ({latest_git_build_number}) and local ({current_build_number}) repositories are not equal\n'
f'It maybe because this is rebuild of old build for which build number already has been increased\n'
f'Stop operation')
return False
log.info('Increasing build number')
if latest_build_number_path.exists():
try:
log.info(f'\tChanging build numbers file')
with latest_build_number_path.open('r+') as build_number_file:
build_numbers = json.load(build_number_file)
new_build_number = build_numbers[component][branch] + 1
build_numbers[component][branch] = new_build_number
build_number_file.seek(0)
build_number_file.write(
json.dumps(build_numbers, indent=4, sort_keys=True))
build_number_file.truncate()
log.info(f'\tPush changes')
push_change_commands = [
'git add -A',
f'git commit -m "Increased build number of {branch} branch for {component} to {new_build_number}"',
f'git push origin HEAD:{branch}'
]
for command in push_change_commands:
return_code, output = cmd_exec(command,
cwd=latest_version_repo_path)
if return_code:
log.error(output)
except Exception:
log.exception('Exception occurred')
return False
else:
log.error(
f'Increasing build number failed, because {latest_build_number_path} does not exist'
)
return False
log.info(f'Build number was increased to {new_build_number}')
return True
def _check_psnr(self):
"""
Check PSNR consistence
:return: Boolean
"""
self.log.info('-' * 80)
self.log.info('Check PSNR')
psnr = None
_output_file = pathlib.Path(self._output_file)
if not _output_file.exists():
self.log.error(f'{_output_file} does not exist')
self.log.error(f'reference psnr: {self._ref_value}')
self.log.error(f'actual psnr: {psnr}')
return False
cmd_copy = copy(self._cmd)
if self._feature == 'decode':
self.log.info('Decode the input file with ffmpeg sw')
sw_output_file = _output_file.parent / f'{_output_file.name}_sw.yuv'
# remove -hwaccel
idx = cmd_copy.index('-hwaccel')
cmd_copy.pop(idx)
cmd_copy.pop(idx)
# remove -hwaccel_device
idx = cmd_copy.index('-hwaccel_device')
cmd_copy.pop(idx)
cmd_copy.pop(idx)
# update output file
cmd_copy[-1] = str(sw_output_file)
# execute ffmpeg sw decode
code, out = cmd_exec(cmd_copy, shell=False, log=self.log)
if code:
self.log.error(out)
return False
self.log.info(out)
resolution = re.search(rf'.*Stream #.*, (\d*x\d*).*',
out).group(1).strip()
self._width, self._height = resolution.split('x')
# compare outputs
if not self._compare_files(self._output_file, sw_output_file):
psnr = self._get_psnr(sw_output_file, self._output_file,
self._width, self._height)
elif self._feature == 'encode':
self.log.info('Encode the input file with ffmpeg sw')
output_yuv = _output_file.parent / f'{_output_file.name}.yuv'
# get -vframes
vframes = cmd_copy[cmd_copy.index('-vframes') + 1]
ffmpeg_cmd = [
'ffmpeg', '-v', 'debug', '-i', self._output_file, '-pix_fmt',
'yuv420p', '-f', 'rawvideo', '-vsync', 'passthrough',
'-vframes', vframes, '-y',
str(output_yuv)
]
code, out = cmd_exec(ffmpeg_cmd, shell=False, log=self.log)
if code:
self.log.error(out)
return False
self.log.info(out)
psnr = self._get_psnr(self._input_file, output_yuv, self._width,
self._height)
elif self._feature == 'vp':
self.log.info('Scale the input file with ffmpeg sw')
if '-vf' in cmd_copy:
vf_args = cmd_copy[cmd_copy.index('-vf') + 1]
if 'scale_vaapi' in vf_args:
sw_output_file = _output_file.parent / f'{_output_file.name}_sw.yuv'
self._width, self._height = re.search(
r'.*scale_vaapi=w=(\d+):h=(\d+).*', vf_args).groups()
self.log.info(
f'Scale: width = {self._width}; height = {self._height}'
)
# remove -hwaccel
idx = cmd_copy.index('-hwaccel')
cmd_copy.pop(idx)
cmd_copy.pop(idx)
# remove -vaapi_device
idx = cmd_copy.index('-vaapi_device')
cmd_copy.pop(idx)
cmd_copy.pop(idx)
# update -vf
cmd_copy[cmd_copy.index('-vf') +
1] = f'scale={self._width}:{self._height}'
# update output file
cmd_copy[-1] = str(sw_output_file)
# scaling with ffmpeg sw
code, out = cmd_exec(cmd_copy, shell=False, log=self.log)
if code:
self.log.error(out)
return False
self.log.info(out)
# compare outputs
if not self._compare_files(self._output_file,
sw_output_file):
psnr = self._get_psnr(sw_output_file,
self._output_file, self._width,
self._height)
else:
self.log.error(f'Feature {self._feature} is not supported')
return False
if psnr is None:
return True
if not psnr:
return False
self.log.info(f'reference psnr: {self._ref_value}')
self.log.info(f'actual psnr: {psnr}')
psnr_gap = 100 * (float(psnr) - float(self._ref_value)) / float(
self._ref_value)
psnr_gap = round(psnr_gap, 4)
self.log.info(f'psnr gap: {psnr_gap}%')
if psnr_gap < -5:
return False
return True
def run(self, options=None):
"""
Script runner
:return: None | subprocess.CalledProcessError
"""
self.log.info('-' * 50)
if self.callfunc:
#TODO: Modify and restore ENV for callfunc too
func, args, kwargs = self.callfunc
self.log.info('function: %s', func)
if args:
self.log.info('args: %s', args)
if kwargs:
self.log.info('kwargs: %s', kwargs)
try:
return_value = func(*args, **kwargs)
return_msg = f'The function returned: {return_value}'
if isinstance(return_value, bool) and return_value == False:
self.log.error(return_msg)
error_code = ErrorCode.CRITICAL.value
else:
self.log.info(return_msg)
error_code = ErrorCode.SUCCESS.value
except Exception:
error_code = ErrorCode.CRITICAL.value
self.log.exception('Failed to call the function:')
elif self.cmd:
if isinstance(self.cmd, list):
self.cmd = ' && '.join(self.cmd)
env = os.environ.copy()
if options:
self.cmd = self.cmd.format_map(options)
if options.get('ENV'):
env.update(options['ENV'])
if self.env:
env.update(self.env)
if self.work_dir:
self.work_dir.mkdir(parents=True, exist_ok=True)
error_code, out = cmd_exec(self.cmd,
env=env,
cwd=self.work_dir,
log=self.log)
if error_code:
self._parse_logs(out)
else:
if self.verbose:
self.log.info(out)
self.log.info('completed')
else:
self.log.debug(out)
self.log.debug('completed')
else:
error_code = ErrorCode.CRITICAL.value
self.log.critical(
f'The action "{self.name}" has not cmd or callfunc parameters')
return error_code
