def init_driver_test_factory(self, test_specification, props):
test_factory = self.factory_with_deploying_infrastructure_step(props)
worker_os = props['os']
get_path = bb.utils.get_path_on_os(worker_os)
branch = props.getProperty('target_branch') or props.getProperty(
'branch')
driver_manifest_path = MediaSdkDirectories.get_build_dir(
branch,
"pre_commit" if props.hasProperty('target_branch') else 'commit',
props['revision'],
test_specification['product_type'],
test_specification['build_type'],
product='media-driver')
command = [
self.run_command[worker_os], "tests_runner.py", '--artifacts',
str(driver_manifest_path), '--root-dir',
util.Interpolate('%(prop:builddir)s/test_dir'), '--stage'
]
for test_stage in TestStage:
test_factory.append(
steps.ShellCommand(
name=test_stage.value,
command=command + [test_stage.value],
workdir=get_path(r"infrastructure/build_scripts")))
return test_factory
def install_components(manifest, components):
"""
:param manifest: Path to a manifest file
:type manifest: String | Manifest
:param components: List of components to install
:type components: List
:return: Boolean
"""
if not isinstance(manifest, Manifest):
manifest = Manifest(manifest)
pkg_type = get_pkg_type()
log = logging.getLogger('install_components')
for component in components:
log.info(f'Installing component: {component}')
comp = manifest.get_component(component)
if not comp:
log.error(f'{component} does not exist in manifest')
return False
repo = comp.trigger_repository
artifacts = MediaSdkDirectories.get_build_dir(
branch=repo.target_branch if repo.target_branch else repo.branch,
build_event=Build_event.PRE_COMMIT.value
if repo.target_branch else Build_event.COMMIT.value,
commit_id=repo.revision,
product_type=comp.product_type,
build_type='release',
product=component)
packages = [
pkg_path for pkg_path in artifacts.glob(f'*.{pkg_type}')
if component in pkg_path.name.lower()
]
# TODO: solve situation with multiple packages installation, e.g. "package" and "package-devel"
if len(packages) > 1:
log.info(
f'Found multiple "{component}" packages {packages} in {artifacts}'
)
return False
if len(packages) == 0:
log.info(f'Package "{component}" was not found in {artifacts}')
return False
if not package_manager.uninstall_pkg(component):
return False
if not package_manager.install_pkg(packages[0]):
return False
return True
def _get_dependencies(self):
deps = self._config_variables.get("DEPENDENCIES", {})
if not deps:
return True
try:
deps_dir = self._options['DEPENDENCIES_DIR']
self._log.info(
f'Dependencies was found. Trying to extract to {deps_dir}')
deps_dir.mkdir(parents=True, exist_ok=True)
self._log.info(f'Creating manifest')
for dependency in deps:
self._log.info(f'Getting component {dependency}')
comp = self._manifest.get_component(dependency)
if comp:
trigger_repo = comp.trigger_repository
if trigger_repo:
dep_dir = MediaSdkDirectories.get_build_dir(
trigger_repo.target_branch
if trigger_repo.target_branch else
trigger_repo.branch,
Build_event.COMMIT.value,
trigger_repo.revision,
comp.product_type,
Build_type.RELEASE.value,
product=dependency)
try:
self._log.info(
f'Extracting {dependency} {comp.product_type} artifacts'
)
# TODO: Extension hardcoded for open source. Need to use only .zip in future.
extract_archive(dep_dir / f'install_pkg.tar.gz',
deps_dir / dependency)
except Exception:
self._log.exception('Can not extract archive')
return False
else:
self._log.error('There is no repository as a trigger')
return False
else:
self._log.error(
f'Component {dependency} does not exist in manifest')
return False
except Exception:
self._log.exception('Exception occurred:')
return False
return True
def check_component_existence(path_to_manifest, component_name):
log = logging.getLogger('component_checker')
log.info(f"Getting data for {component_name} from {path_to_manifest}")
manifest = Manifest(pathlib.Path(path_to_manifest))
component = manifest.get_component(component_name)
repository = component.get_repository(component_name)
component_dir = MediaSdkDirectories.get_build_dir(
repository.target_branch if repository.target_branch else repository.branch,
Build_event.COMMIT.value,
repository.revision,
component.product_type,
Build_type.RELEASE.value,
product=component_name)
if component_dir.exists():
log.info(f"Directory {component_dir} exists")
# This is stop phrase for buildbot to skip all build stages
log.info(SKIP_BUILDING_DEPENDENCY_PHRASE)
else:
log.info(f"Directory {component_dir} doesn't exist")
def main():
"""
Tests runner
:return: None
"""
parser = argparse.ArgumentParser(
prog="test_adapter.py", formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("--version", action="version", version="%(prog)s 1.0")
parser.add_argument('-br',
"--branch",
metavar="String",
required=True,
help="Branch of triggered repository")
parser.add_argument(
'-e',
"--build-event",
default='commit',
choices=[build_event.value for build_event in Build_event],
help='Event of commit')
parser.add_argument('-c',
"--commit-id",
metavar="String",
required=True,
help="SHA of triggered commit")
parser.add_argument(
'-p',
"--product-type",
default='closed_linux',
choices=[product_type.value for product_type in Product_type],
help='Type of product')
parser.add_argument(
'-b',
"--build-type",
default='release',
choices=[build_type.value for build_type in Build_type],
help='Type of build')
parser.add_argument('-d',
"--root-dir",
metavar="PATH",
required=True,
help="Path to worker directory")
args = parser.parse_args()
directories_layout = {
'branch': args.branch,
'build_event': args.build_event,
'commit_id': args.commit_id,
'product_type': args.product_type,
'build_type': args.build_type,
}
build_artifacts_dir = MediaSdkDirectories.get_build_dir(
**directories_layout)
tests_artifacts_dir = MediaSdkDirectories.get_test_dir(
**directories_layout)
tests_artifacts_url = MediaSdkDirectories.get_test_url(
**directories_layout)
log = logging.getLogger('test_adapter.log')
adapter = TedAdapter(build_artifacts_dir,
tests_artifacts_dir,
tests_artifacts_url,
root_dir=pathlib.Path(args.root_dir))
# Install third parties for msdk
if not adapter.install_pkgs(THIRD_PARTY):
log.info(f'Required packages "{THIRD_PARTY}" were not installed\n')
exit(TestReturnCodes.INFRASTRUCTURE_ERROR.value)
# Check existence of driver
check_driver()
# Install msdk
if not adapter.install_pkgs(['mediasdk'], clean_dir=True):
log.info(f'Package "mediasdk" was not installed\n')
exit(TestReturnCodes.INFRASTRUCTURE_ERROR.value)
try:
tests_return_code = adapter.run_test()
except Exception:
print("Exception occurred:\n", traceback.format_exc())
# TODO return json string
tests_return_code = TestReturnCodes.INFRASTRUCTURE_ERROR.value
try:
tests_return_code |= adapter.run_fei_tests()
except Exception:
print("Exception occurred:\n", traceback.format_exc())
# TODO return json string
tests_return_code |= TestReturnCodes.INFRASTRUCTURE_ERROR.value
try:
adapter.copy_logs_to_share()
except Exception:
print("Exception occurred while copying results:\n",
traceback.format_exc())
tests_return_code |= TestReturnCodes.INFRASTRUCTURE_ERROR.value
exit(tests_return_code)
def _copy(self):
"""
Copy 'pack' stage results to share folder
:return: None | Exception
"""
print('-' * 50)
self._log.info("COPYING")
branch = 'unknown'
commit_id = 'unknown'
if self._changed_repo:
repo_name, branch, commit_id = self._changed_repo.split(':')
if self._target_branch:
branch = self._target_branch
if commit_id == 'HEAD':
commit_id = ProductState.get_head_revision(
self._options['REPOS_DIR'] / repo_name)
elif self._repo_states:
for repo in self._repo_states.values():
if repo['trigger']:
branch = repo['target_branch'] if repo.get(
'target_branch') else repo['branch']
commit_id = repo['commit_id']
elif self._manifest_file:
component = self._manifest.get_component(self._product)
repo = component.trigger_repository
branch = repo.target_branch if repo.target_branch else repo.branch
commit_id = repo.revision
else:
# "--changed-repo" or "--repo-states" arguments are not set, "HEAD" revision and "master" branch are used
branch = 'master'
commit_id = 'HEAD'
build_dir = MediaSdkDirectories.get_build_dir(
branch,
self._build_event,
commit_id,
self._product_type,
self._options["BUILD_TYPE"],
product=self._product)
build_url = MediaSdkDirectories.get_build_url(
branch,
self._build_event,
commit_id,
self._product_type,
self._options["BUILD_TYPE"],
product=self._product)
build_root_dir = MediaSdkDirectories.get_root_builds_dir()
rotate_dir(build_dir)
self._log.info('Copy to %s', build_dir)
self._log.info('Artifacts are available by: %s', build_url)
# Workaround for copying to samba share on Linux
# to avoid exceptions while setting Linux permissions.
_orig_copystat = shutil.copystat
shutil.copystat = lambda x, y, follow_symlinks=True: x
shutil.copytree(self._options['PACK_DIR'], build_dir)
shutil.copystat = _orig_copystat
if not self._run_build_config_actions(Stage.COPY.value):
return False
if self._build_state_file.exists():
with self._build_state_file.open() as state:
build_state = json.load(state)
if build_state['status'] == "PASS":
last_build_path = build_dir.relative_to(build_root_dir)
last_build_file = build_dir.parent.parent / f'last_build_{self._product_type}'
last_build_file.write_text(str(last_build_path))
return True