def extract_all_repos(self):
"""
Get repositories and checkout them to the right state
:return: None
"""
git_commit_date = None
for repo in self.repo_states:
if repo.commit_id != 'HEAD':
repo.prepare_repo()
if MediaSdkDirectories.is_release_branch(repo.branch_name):
if not repo.is_branch_exist(repo.branch_name):
raise BranchDoesNotExistException(
f'Release branch {repo.branch_name} does not exist in the repo {repo.repo_name}'
)
repo.change_repo_state(branch_name=repo.branch_name)
else:
repo.change_repo_state()
if repo.is_trigger:
git_commit_date = repo.get_time()
commit_timestamp = self.commit_time.timestamp() \
if self.commit_time \
else git_commit_date
for repo in self.repo_states:
if repo.commit_id == 'HEAD':
repo.prepare_repo()
if MediaSdkDirectories.is_release_branch(repo.branch_name):
if not repo.is_branch_exist(repo.branch_name):
raise BranchDoesNotExistException(
f'Release branch {repo.branch_name} does not exist in the repo {repo.repo_name}'
)
repo.change_repo_state(branch_name=repo.branch_name,
commit_time=commit_timestamp)
# if parameters '--commit-time', '--changed-repo' and '--repo-states' didn't set
# then variable 'commit_timestamp' is 'None' and 'HEAD' revisions be used
else:
repo.change_repo_state(commit_time=commit_timestamp)
def extract_repo(root_dir,
repo_name,
url=None,
branch='master',
commit_id='HEAD',
commit_time=None,
proxy=False):
"""
Prepare repository
:param root_dir: Directory where repositories will clone
:param repo_name: Name of repository
:param url: Url to repository
:param branch: Branch of repository
:param commit_id: Revision of commit
:param commit_time: Time for getting slice of commits of repositories
:param proxy: Proxy enabling
:return: None | Exception
"""
log = logging.getLogger('ProductState.extract_repo')
if url is None:
url = MediaSdkDirectories.get_repo_url_by_name(repo_name)
root_dir = pathlib.Path(root_dir)
root_dir.mkdir(parents=True, exist_ok=True)
if commit_time and not isinstance(commit_time, float):
commit_time = datetime.strptime(commit_time,
'%Y-%m-%d %H:%M:%S').timestamp()
if not commit_time and not commit_id:
log.info(
'Commit id and timestamp not specified, clone HEAD of repository')
try:
repo = GitRepo(root_repo_dir=root_dir,
repo_name=repo_name,
branch=branch,
url=url,
commit_id=commit_id)
repo.prepare_repo()
repo.change_repo_state(commit_time=commit_time)
except Exception:
log.exception('Exception occurred')
log.info("EXTRACTING FAILED")
exit(ErrorCode.CRITICAL.value)
log.info("EXTRACTING COMPLETED")
def is_release_branch(raw_branch):
# TODO: need to unify with MediaSdkDirectories.is_release_branch method
"""
Checks if branch is release branch
Used as branch filter for pollers
"""
# Ignore pull request branches like 'refs/pull/*'
if raw_branch.startswith('refs/heads/'):
branch = raw_branch[11:]
if MediaSdkDirectories.is_release_branch(branch) or branch == 'master':
return True
return False
def _get_dependencies(self):
deps = self._config_variables.get("DEPENDENCIES", {})
if not deps:
return True
try:
deps_dir = self._options['DEPENDENCIES_DIR']
self._log.info(
f'Dependencies was found. Trying to extract to {deps_dir}')
deps_dir.mkdir(parents=True, exist_ok=True)
self._log.info(f'Creating manifest')
for dependency in deps:
self._log.info(f'Getting component {dependency}')
comp = self._manifest.get_component(dependency)
if comp:
trigger_repo = comp.trigger_repository
if trigger_repo:
dep_dir = MediaSdkDirectories.get_build_dir(
trigger_repo.target_branch
if trigger_repo.target_branch else
trigger_repo.branch,
Build_event.COMMIT.value,
trigger_repo.revision,
comp.product_type,
Build_type.RELEASE.value,
product=dependency)
try:
self._log.info(
f'Extracting {dependency} {comp.product_type} artifacts'
)
# TODO: Extension hardcoded for open source. Need to use only .zip in future.
extract_archive(dep_dir / f'install_pkg.tar.gz',
deps_dir / dependency)
except Exception:
self._log.exception('Can not extract archive')
return False
else:
self._log.error('There is no repository as a trigger')
return False
else:
self._log.error(
f'Component {dependency} does not exist in manifest')
return False
except Exception:
self._log.exception('Exception occurred:')
return False
return True
def get_manifest_path(self, props):
"""
Return path to main manifest on share as string for target worker.
get_path and util.Interpolate wrappers are not needed.
"""
return str(
MediaSdkDirectories.get_commit_dir(
props.getProperty('target_branch')
or props.getProperty('branch'),
props.getProperty('event_type'),
props.getProperty("revision"),
os_type=props.getProperty('os'),
# TODO: import the const from common
product='manifest') / 'manifest.yml')
def is_limited_number_of_commits(pull_request, token=None):
"""
Checks if number of commits does not exceed the specified value
"""
github_commits_url = pull_request['commits_url']
data = get_data(github_commits_url,
additional_headers={'Authorization': f'token {token}'}
if token else None)
number_of_commits = len(data)
branch = pull_request['base']['ref']
if number_of_commits > MAX_NUM_COMMITS:
return False
if number_of_commits > MAX_NUM_COMMITS_RELEASE_BRANCH and \
(MediaSdkDirectories.is_release_branch(branch) or branch == 'master'):
return False
return True
def extract_open_source_infrastructure(root_dir, branch, commit_id,
commit_time, manifest):
log = logging.getLogger('extract_repo.extract_open_source_infrastructure')
repos = MediaSdkDirectories()
open_source_product_configs_repo = repos.open_source_product_configs_repo
open_source_infra_repo = repos.open_source_infrastructure_repo
# Extract product configs
if not manifest:
extract_repo(root_repo_dir=root_dir,
repo_name=open_source_product_configs_repo,
branch=branch,
commit_id=commit_id,
commit_time=commit_time)
manifest_data = Manifest(root_dir / open_source_product_configs_repo /
'manifest.yml')
else:
manifest_data = Manifest(manifest)
product_conf = manifest_data.get_component('infra').get_repository(
open_source_product_configs_repo)
extract_repo(root_repo_dir=root_dir,
repo_name=product_conf.name,
branch=product_conf.branch,
commit_id=product_conf.revision,
commit_time=commit_time)
open_source_infra = manifest_data.get_component('infra').get_repository(
open_source_infra_repo)
# Extract open source infrastructure
extract_repo(root_repo_dir=root_dir,
repo_name=open_source_infra.name,
branch=open_source_infra.branch,
commit_id=open_source_infra.revision)
try:
log.info(f"Copy secrets")
shutil.copyfile(
str(pathlib.Path('msdk_secrets.py').absolute()),
str(root_dir / open_source_infra_repo / 'common' /
'msdk_secrets.py'))
except Exception:
log.exception('Can not create infrastructure package')
# TODO: An exit from script should be in main()
exit_script(ErrorCode.CRITICAL)
def generate_build_config(self):
"""
Build configuration file parser
:return: None | Exception
"""
global_vars = {
'action': self._action,
'vs_component': self._vs_component,
'options': self.options,
'stage': Stage,
'copy_win_files': copy_win_files,
'args': self.custom_cli_args,
'log': self.log,
'product_type': self.product_type,
'build_event': self.build_event,
# TODO should be in lower case
'DEV_PKG_DATA_TO_ARCHIVE': self.dev_pkg_data_to_archive,
'INSTALL_PKG_DATA_TO_ARCHIVE': self.install_pkg_data_to_archive,
'get_build_number': get_build_number,
'get_api_version': self._get_api_version,
'branch_name': self.branch_name,
'update_config': self._update_config,
'target_arch': self.target_arch,
'commit_time': self.commit_time
}
exec(
open(self.build_config_path).read(), global_vars,
self.config_variables)
# TODO add product_repos to global_vars
if 'PRODUCT_REPOS' in self.config_variables:
for repo in self.config_variables['PRODUCT_REPOS']:
self.product_repos[repo['name']] = {
'branch': repo.get('branch'),
'commit_id': repo.get('commit_id'),
'url':
MediaSdkDirectories.get_repo_url_by_name(repo['name'])
}
return True
def check_component_existence(path_to_manifest, component_name):
log = logging.getLogger('component_checker')
log.info(f"Getting data for {component_name} from {path_to_manifest}")
manifest = Manifest(pathlib.Path(path_to_manifest))
component = manifest.get_component(component_name)
repository = component.get_repository(component_name)
component_dir = MediaSdkDirectories.get_build_dir(
repository.target_branch if repository.target_branch else repository.branch,
Build_event.COMMIT.value,
repository.revision,
component.product_type,
Build_type.RELEASE.value,
product=component_name)
if component_dir.exists():
log.info(f"Directory {component_dir} exists")
# This is stop phrase for buildbot to skip all build stages
log.info(SKIP_BUILDING_DEPENDENCY_PHRASE)
else:
log.info(f"Directory {component_dir} doesn't exist")
def _check_branch(self):
"""
Check release branch
"""
self._log.info('Checking release branch')
if self._target_branch:
branch_to_check = self._target_branch
else:
branch_to_check = self._branch
if MediaSdkDirectories.is_release_branch(branch_to_check):
sdk_br, driver_br = convert_branch(branch_to_check)
for repo_name in self._release_repos:
if repo_name == 'media-driver':
self._release_branch[repo_name] = driver_br
else:
self._release_branch[repo_name] = sdk_br
"mediasdk-api-next": {
"factory": FACTORIES.init_build_factory,
"product_conf_file": "conf_linux_public.py",
"product_type": Product_type.PUBLIC_LINUX_API_NEXT.value,
"build_type": Build_type.RELEASE.value,
"api_latest": True,
"fastboot": False,
"compiler": "gcc",
"compiler_version": "6.3.1",
"worker": "centos",
"dependency_name": 'mediasdk',
# Builder is enabled for not release branches
'triggers': [{'builders': ['libva'],
'filter': GithubCommitFilter(
PRODUCTION_REPOS,
lambda branch, target_branch: not MediaSdkDirectories.is_release_branch(
target_branch or branch))}]
},
"mediasdk-gcc-9.2.1": {
"factory": FACTORIES.init_build_factory,
"product_conf_file": "conf_linux_public.py",
"product_type": Product_type.PUBLIC_LINUX_GCC_LATEST.value,
"build_type": Build_type.RELEASE.value,
"api_latest": False,
"fastboot": False,
"compiler": "gcc",
"compiler_version": "9.2.1",
"worker": "ubuntu",
"dependency_name": 'mediasdk',
'triggers': [{'builders': ['libva'],
'filter': GithubCommitFilter(
def extract_repo(root_repo_dir,
repo_name,
branch,
commit_id=None,
commit_time=None,
proxy=False):
log = logging.getLogger('extract_repo.extract_repo')
try:
repo_url = MediaSdkDirectories.get_repo_url_by_name(repo_name)
if commit_id:
repo = git_worker.GitRepo(root_repo_dir=root_repo_dir,
repo_name=repo_name,
branch=branch,
url=repo_url,
commit_id=commit_id)
repo.prepare_repo()
repo.change_repo_state()
elif commit_time:
repo = git_worker.GitRepo(root_repo_dir=root_repo_dir,
repo_name=repo_name,
branch='master',
url=repo_url)
repo.prepare_repo()
if MediaSdkDirectories.is_release_branch(branch):
if not repo.is_branch_exist(branch):
raise git_worker.BranchDoesNotExistException(
f'Release branch {branch} does not exist in the repo {repo.repo_name}'
)
# repo.branch = branch
repo.change_repo_state(branch_name=branch,
commit_time=datetime.strptime(
commit_time,
'%Y-%m-%d %H:%M:%S').timestamp())
else:
repo.change_repo_state(commit_time=datetime.strptime(
commit_time, '%Y-%m-%d %H:%M:%S').timestamp())
else:
log.info(
'Commit id and timestamp not specified, clone HEAD of repository'
)
repo = git_worker.GitRepo(root_repo_dir=root_repo_dir,
repo_name=repo_name,
branch='master',
url=repo_url)
repo.prepare_repo()
if MediaSdkDirectories.is_release_branch(branch):
if not repo.is_branch_exist(branch):
raise git_worker.BranchDoesNotExistException(
f'Release branch {branch} does not exist in the repo {repo.repo_name}'
)
# repo.branch = branch
repo.change_repo_state(branch_name=branch)
else:
# repo.branch = master
repo.change_repo_state()
except Exception:
log.exception('Exception occurred')
exit_script(ErrorCode.CRITICAL)
def extract_private_infrastructure(root_dir, branch, commit_id, commit_time):
log = logging.getLogger('extract_repo.extract_private_infrastructure')
infrastructure_root_dir = root_dir / 'infrastructure'
# We save and update repos in temporary folder and create infrastructure package from it
# So, not needed extracting repo to the beginning each time
original_repos_dir = root_dir / 'tmp_infrastructure'
repos = MediaSdkDirectories()
open_source_product_configs_repo = repos.open_source_product_configs_repo
open_source_infra_repo = repos.open_source_infrastructure_repo
closed_source_product_configs_repo = repos.closed_source_product_configs_repo
closed_source_infra_repo = repos.closed_source_infrastructure_repo
# Extract open source infrastructure and product configs
extract_open_source_infrastructure(original_repos_dir, branch, commit_id,
commit_time)
# Extract closed source product configs
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_product_configs_repo,
branch='master',
commit_time=commit_time)
# Get revision of closed source infrastructure from closed source product configs repo
configs_dir = original_repos_dir / closed_source_product_configs_repo
sys.path.append(str(configs_dir))
import infrastructure_version
infrastructure_version = reload(infrastructure_version)
closed_source_infra_version = infrastructure_version.CLOSED_SOURCE
# Extract closed source infrastructure
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_infra_repo,
branch=closed_source_infra_version['branch'],
commit_id=closed_source_infra_version['commit_id'])
log.info('-' * 50)
log.info(f"Create infrastructure package")
try:
log.info(f"- Delete existing infrastructure")
if infrastructure_root_dir.exists():
remove_directory(str(infrastructure_root_dir))
log.info(f"- Copy open source infrastructure")
copy_tree(str(original_repos_dir / open_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Copy closed source infrastructure")
copy_tree(str(original_repos_dir / closed_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Remove closed source static data")
(infrastructure_root_dir / 'common' / 'static_closed_data.py').unlink()
log.info(f"- Copy open source product configs")
copy_tree(
str(original_repos_dir / open_source_product_configs_repo),
str(infrastructure_root_dir / open_source_product_configs_repo))
log.info(f"- Copy closed source product configs")
copy_tree(
str(original_repos_dir / closed_source_product_configs_repo),
str(infrastructure_root_dir / open_source_product_configs_repo))
# log.info(f"Copy secrets")
shutil.copyfile(
str(pathlib.Path('msdk_secrets.py').absolute()),
str(infrastructure_root_dir / 'common' / 'msdk_secrets.py'))
except Exception:
log.exception('Can not create infrastructure package')
exit_script(ErrorCode.CRITICAL)
def extract_private_infrastructure(root_dir, branch, commit_id, commit_time,
manifest):
log = logging.getLogger('extract_repo.extract_private_infrastructure')
infrastructure_root_dir = root_dir / 'infrastructure'
configs_root_dir = root_dir / 'product-configs'
# We save and update repos in temporary folder and create infrastructure package from it
# So, not needed extracting repo to the beginning each time
original_repos_dir = root_dir / 'tmp_infrastructure'
repos = MediaSdkDirectories()
open_source_product_configs_repo = repos.open_source_product_configs_repo
open_source_infra_repo = repos.open_source_infrastructure_repo
closed_source_product_configs_repo = repos.closed_source_product_configs_repo
closed_source_infra_repo = repos.closed_source_infrastructure_repo
# Extract open source infrastructure and product configs
extract_open_source_infrastructure(original_repos_dir, branch, commit_id,
commit_time, manifest)
# Extract closed source product configs
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_product_configs_repo,
branch='master',
commit_time=commit_time)
manifest_path = original_repos_dir / closed_source_product_configs_repo / 'manifest.yml'
manifest_data = Manifest(manifest_path)
closed_source_infra = manifest_data.get_component('infra').get_repository(
closed_source_infra_repo)
# Extract closed source infrastructure
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_infra.name,
branch=closed_source_infra.branch,
commit_id=closed_source_infra.revision)
# Event repository in infra component for private builds is product-configs,
# so need to change default trigger value
manifest_data.get_component('infra').build_info.set_trigger(
open_source_product_configs_repo)
manifest_data.save_manifest(manifest_path)
log.info('-' * 50)
log.info(f"Create infrastructure package")
try:
log.info(f"- Delete existing infrastructure")
if infrastructure_root_dir.exists():
remove_directory(str(infrastructure_root_dir))
if configs_root_dir.exists():
remove_directory(str(configs_root_dir))
log.info(f"- Copy open source infrastructure")
copy_tree(str(original_repos_dir / open_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Copy closed source infrastructure")
copy_tree(str(original_repos_dir / closed_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Remove closed source static data")
(infrastructure_root_dir / 'common' / 'static_closed_data.py').unlink()
log.info(f"- Copy open source product configs")
copy_tree(str(original_repos_dir / open_source_product_configs_repo),
str(configs_root_dir))
log.info(f"- Copy closed source product configs")
copy_tree(str(original_repos_dir / closed_source_product_configs_repo),
str(configs_root_dir))
# log.info(f"Copy secrets")
shutil.copyfile(
str(pathlib.Path('msdk_secrets.py').absolute()),
str(infrastructure_root_dir / 'common' / 'msdk_secrets.py'))
except Exception:
log.exception('Can not create infrastructure package')
exit_script(ErrorCode.CRITICAL)
def extract_closed_source_infrastructure(root_dir, branch, commit_id,
commit_time, manifest):
log = logging.getLogger(
'extract_repo.extract_closed_source_infrastructure')
infrastructure_root_dir = root_dir / 'infrastructure'
configs_root_dir = root_dir / 'product-configs'
# We save and update repos in temporary folder and create infrastructure package from it
# So, not needed extracting repo to the beginning each time
original_repos_dir = root_dir / 'tmp_infrastructure'
repos = MediaSdkDirectories()
closed_source_product_configs_repo = repos.closed_source_product_configs_repo
open_source_infra_repo = repos.open_source_infrastructure_repo
closed_source_infra_repo = repos.closed_source_infrastructure_repo
# Extract product configs
if not manifest:
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_product_configs_repo,
branch=branch,
commit_id=commit_id,
commit_time=commit_time)
manifest_data = Manifest(original_repos_dir /
closed_source_product_configs_repo /
'manifest.yml')
else:
manifest_data = Manifest(manifest)
product_conf = manifest_data.get_component('infra').get_repository(
closed_source_product_configs_repo)
extract_repo(root_repo_dir=original_repos_dir,
repo_name=product_conf.name,
branch=product_conf.branch,
commit_id=product_conf.revision,
commit_time=commit_time)
open_source_infra = manifest_data.get_component('infra').get_repository(
open_source_infra_repo)
closed_source_infra = manifest_data.get_component('infra').get_repository(
closed_source_infra_repo)
# Extract open source infrastructure
# Set proxy for access to GitHub
extract_repo(root_repo_dir=original_repos_dir,
repo_name=open_source_infra.name,
branch=open_source_infra.branch,
commit_id=open_source_infra.revision,
proxy=True)
# Extract closed source part of infrastructure
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_infra.name,
branch=closed_source_infra.branch,
commit_id=closed_source_infra.revision)
log.info('-' * 50)
log.info(f"Create infrastructure package")
try:
log.info(f"- Delete existing infrastructure")
if infrastructure_root_dir.exists():
remove_directory(str(infrastructure_root_dir))
if configs_root_dir.exists():
remove_directory(str(configs_root_dir))
log.info(f"- Copy open source infrastructure")
copy_tree(str(original_repos_dir / open_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Copy closed source infrastructure")
copy_tree(str(original_repos_dir / closed_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Copy product configs")
copy_tree(str(original_repos_dir / closed_source_product_configs_repo),
str(configs_root_dir))
# log.info(f"Copy secrets")
shutil.copyfile(
str(pathlib.Path('msdk_secrets.py').absolute()),
str(infrastructure_root_dir / 'common' / 'msdk_secrets.py'))
except Exception:
log.exception('Can not create infrastructure package')
exit_script(ErrorCode.CRITICAL)
def main():
"""
Tests runner
:return: None
"""
parser = argparse.ArgumentParser(
prog="test_adapter.py", formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("--version", action="version", version="%(prog)s 1.0")
parser.add_argument('-br',
"--branch",
metavar="String",
required=True,
help="Branch of triggered repository")
parser.add_argument(
'-e',
"--build-event",
default='commit',
choices=[build_event.value for build_event in Build_event],
help='Event of commit')
parser.add_argument('-c',
"--commit-id",
metavar="String",
required=True,
help="SHA of triggered commit")
parser.add_argument(
'-p',
"--product-type",
default='closed_linux',
choices=[product_type.value for product_type in Product_type],
help='Type of product')
parser.add_argument(
'-b',
"--build-type",
default='release',
choices=[build_type.value for build_type in Build_type],
help='Type of build')
parser.add_argument('-d',
"--root-dir",
metavar="PATH",
required=True,
help="Path to worker directory")
args = parser.parse_args()
directories_layout = {
'branch': args.branch,
'build_event': args.build_event,
'commit_id': args.commit_id,
'product_type': args.product_type,
'build_type': args.build_type,
}
build_artifacts_dir = MediaSdkDirectories.get_build_dir(
**directories_layout)
tests_artifacts_dir = MediaSdkDirectories.get_test_dir(
**directories_layout)
tests_artifacts_url = MediaSdkDirectories.get_test_url(
**directories_layout)
log = logging.getLogger('test_adapter.log')
adapter = TedAdapter(build_artifacts_dir,
tests_artifacts_dir,
tests_artifacts_url,
root_dir=pathlib.Path(args.root_dir))
# Install third parties for msdk
if not adapter.install_pkgs(THIRD_PARTY):
log.info(f'Required packages "{THIRD_PARTY}" were not installed\n')
exit(TestReturnCodes.INFRASTRUCTURE_ERROR.value)
# Check existence of driver
check_driver()
# Install msdk
if not adapter.install_pkgs(['mediasdk'], clean_dir=True):
log.info(f'Package "mediasdk" was not installed\n')
exit(TestReturnCodes.INFRASTRUCTURE_ERROR.value)
try:
tests_return_code = adapter.run_test()
except Exception:
print("Exception occurred:\n", traceback.format_exc())
# TODO return json string
tests_return_code = TestReturnCodes.INFRASTRUCTURE_ERROR.value
try:
tests_return_code |= adapter.run_fei_tests()
except Exception:
print("Exception occurred:\n", traceback.format_exc())
# TODO return json string
tests_return_code |= TestReturnCodes.INFRASTRUCTURE_ERROR.value
try:
adapter.copy_logs_to_share()
except Exception:
print("Exception occurred while copying results:\n",
traceback.format_exc())
tests_return_code |= TestReturnCodes.INFRASTRUCTURE_ERROR.value
exit(tests_return_code)
def _copy(self):
"""
Copy 'pack' stage results to share folder
:return: None | Exception
"""
print('-' * 50)
self._log.info("COPYING")
branch = 'unknown'
commit_id = 'unknown'
if self._changed_repo:
repo_name, branch, commit_id = self._changed_repo.split(':')
if self._target_branch:
branch = self._target_branch
if commit_id == 'HEAD':
commit_id = ProductState.get_head_revision(
self._options['REPOS_DIR'] / repo_name)
elif self._repo_states:
for repo in self._repo_states.values():
if repo['trigger']:
branch = repo['target_branch'] if repo.get(
'target_branch') else repo['branch']
commit_id = repo['commit_id']
elif self._manifest_file:
component = self._manifest.get_component(self._product)
repo = component.trigger_repository
branch = repo.target_branch if repo.target_branch else repo.branch
commit_id = repo.revision
else:
# "--changed-repo" or "--repo-states" arguments are not set, "HEAD" revision and "master" branch are used
branch = 'master'
commit_id = 'HEAD'
build_dir = MediaSdkDirectories.get_build_dir(
branch,
self._build_event,
commit_id,
self._product_type,
self._options["BUILD_TYPE"],
product=self._product)
build_url = MediaSdkDirectories.get_build_url(
branch,
self._build_event,
commit_id,
self._product_type,
self._options["BUILD_TYPE"],
product=self._product)
build_root_dir = MediaSdkDirectories.get_root_builds_dir()
rotate_dir(build_dir)
self._log.info('Copy to %s', build_dir)
self._log.info('Artifacts are available by: %s', build_url)
# Workaround for copying to samba share on Linux
# to avoid exceptions while setting Linux permissions.
_orig_copystat = shutil.copystat
shutil.copystat = lambda x, y, follow_symlinks=True: x
shutil.copytree(self._options['PACK_DIR'], build_dir)
shutil.copystat = _orig_copystat
if not self._run_build_config_actions(Stage.COPY.value):
return False
if self._build_state_file.exists():
with self._build_state_file.open() as state:
build_state = json.load(state)
if build_state['status'] == "PASS":
last_build_path = build_dir.relative_to(build_root_dir)
last_build_file = build_dir.parent.parent / f'last_build_{self._product_type}'
last_build_file.write_text(str(last_build_path))
return True
def _extract(self):
"""
Get and prepare build repositories
Uses git_worker.py module
:return: None | Exception
"""
print('-' * 50)
self._log.info("EXTRACTING")
self._options['REPOS_DIR'].mkdir(parents=True, exist_ok=True)
self._options['PACK_DIR'].mkdir(parents=True, exist_ok=True)
triggered_repo = 'unknown'
if self._changed_repo:
repo_name, branch, commit_id = self._changed_repo.split(':')
triggered_repo = repo_name
if repo_name not in self._product_repos:
self._log.critical(
f'{repo_name} repository is not defined in the product configuration PRODUCT_REPOS'
)
return False
for repo, data in self._product_repos.items():
data['trigger'] = False
if repo == repo_name:
data['trigger'] = True
if self._target_branch:
data['target_branch'] = self._target_branch
if not data.get('branch'):
data['branch'] = branch
data['commit_id'] = commit_id
elif not data.get('branch'):
if self._target_branch and MediaSdkDirectories.is_release_branch(
self._target_branch):
data['branch'] = self._target_branch
elif MediaSdkDirectories.is_release_branch(
branch) and not self._target_branch:
data['branch'] = branch
elif self._repo_states:
for repo_name, values in self._repo_states.items():
if repo_name in self._product_repos:
if values['trigger']:
triggered_repo = repo_name
if values.get('target_branch'):
self._product_repos[repo_name][
'target_branch'] = values['target_branch']
self._product_repos[repo_name]['branch'] = values['branch']
self._product_repos[repo_name]['commit_id'] = values[
'commit_id']
self._product_repos[repo_name]['url'] = values['url']
self._product_repos[repo_name]['trigger'] = values[
'trigger']
elif self._manifest_file:
component = self._manifest.get_component(self._product)
for repo in component.repositories:
if repo.name in self._product_repos:
if repo.is_trigger:
triggered_repo = repo.name
if repo.target_branch:
self._product_repos[repo.name][
'target_branch'] = repo.target_branch
self._product_repos[repo.name]['branch'] = repo.branch
self._product_repos[repo.name]['commit_id'] = repo.revision
self._product_repos[repo.name]['url'] = repo.url
self._product_repos[repo.name]['trigger'] = repo.is_trigger
product_state = ProductState(self._product_repos,
self._options["REPOS_DIR"],
self._commit_time)
product_state.extract_all_repos()
product_state.save_repo_states(self._options["PACK_DIR"] /
'repo_states.json',
trigger=triggered_repo)
self._save_manifest(product_state)
shutil.copyfile(self._config_path,
self._options["PACK_DIR"] / self._config_path.name)
test_scenario = self._config_path.parent / f'{self._config_path.stem}_test{self._config_path.suffix}'
if test_scenario.exists():
shutil.copyfile(test_scenario,
self._options["PACK_DIR"] / test_scenario.name)
if not self._get_dependencies():
return False
if not self._run_build_config_actions(Stage.EXTRACT.value):
return False
return True
def _extract(self):
"""
Get and prepare build repositories
Uses git_worker.py module
:return: None | Exception
"""
print('-' * 50)
self.log.info("EXTRACTING")
self.options['REPOS_DIR'].mkdir(parents=True, exist_ok=True)
self.options['REPOS_FORKED_DIR'].mkdir(parents=True, exist_ok=True)
self.options['PACK_DIR'].mkdir(parents=True, exist_ok=True)
triggered_repo = 'unknown'
if self.changed_repo:
repo_name, branch, commit_id = self.changed_repo.split(':')
triggered_repo = repo_name
if repo_name not in self.product_repos:
self.log.critical(
f'{repo_name} repository is not defined in the product configuration PRODUCT_REPOS'
)
return False
for repo, data in self.product_repos.items():
if repo == repo_name:
if not data.get('branch'):
data['branch'] = branch
data['commit_id'] = commit_id
if self.repo_url:
data['url'] = self.repo_url
elif not data.get('branch'):
if MediaSdkDirectories.is_release_branch(branch):
data['branch'] = branch
elif self.repo_states:
for repo_name, values in self.repo_states.items():
if repo_name in self.product_repos:
if values['trigger']:
triggered_repo = repo_name
self.product_repos[repo_name]['branch'] = values['branch']
self.product_repos[repo_name]['commit_id'] = values[
'commit_id']
self.product_repos[repo_name]['url'] = values['url']
product_state = ProductState(self.product_repos,
self.options["REPOS_DIR"],
self.commit_time)
product_state.extract_all_repos()
product_state.save_repo_states(self.options["PACK_DIR"] /
'repo_states.json',
trigger=triggered_repo)
shutil.copyfile(self.build_config_path,
self.options["PACK_DIR"] / self.build_config_path.name)
if not self._run_build_config_actions(Stage.EXTRACT.value):
return False
return True
"factory":
FACTORIES.init_package_factory,
"worker":
"ubuntu",
'triggers': [{
'builders': ['test-api-next', 'test'],
'filter':
GithubCommitFilter(
PRODUCTION_REPOS, lambda branch, target_branch:
(target_branch or branch) == 'master')
}, {
'builders': ['test'],
'filter':
GithubCommitFilter(
PRODUCTION_REPOS, lambda branch, target_branch:
MediaSdkDirectories.is_release_branch(target_branch or branch))
}]
}
}
FLOW = factories.Flow(BUILDERS, FACTORIES)
WORKERS = {
"windows": {
'b-1-26': {
"os": OsType.windows
},
'b-1-27': {
"os": OsType.windows
},
},
'builders': ['build-libva']}]
},
"build-api-next": {
"factory": FACTORIES.init_build_factory,
"product_conf_file": "conf_linux_public.py",
"product_type": Product_type.PUBLIC_LINUX_API_NEXT.value,
"build_type": Build_type.RELEASE.value,
"api_latest": True,
"fastboot": False,
"compiler": "gcc",
"compiler_version": "6.3.1",
"worker": "centos",
# Builder is enabled for not release branches
'triggers': [{'repositories': PRODUCTION_REPOS,
'branches': lambda branch: not MediaSdkDirectories.is_release_branch(branch),
'builders': ['build-libva']}]
},
"build-gcc-8.2.0": {
"factory": FACTORIES.init_build_factory,
"product_conf_file": "conf_linux_public.py",
"product_type": Product_type.PUBLIC_LINUX_GCC_LATEST.value,
"build_type": Build_type.RELEASE.value,
"api_latest": False,
"fastboot": False,
"compiler": "gcc",
"compiler_version": "8.2.0",
"worker": "ubuntu",
'triggers': [{'repositories': PRODUCTION_REPOS,
'branches': lambda branch: not MediaSdkDirectories.is_release_branch(branch),