def extract_all_repos(self):
"""
Get repositories and checkout them to the right state
:return: None
"""
git_commit_date = None
for repo in self.repo_states:
if repo.commit_id != 'HEAD':
repo.prepare_repo()
if MediaSdkDirectories.is_release_branch(repo.branch_name):
if not repo.is_branch_exist(repo.branch_name):
raise BranchDoesNotExistException(
f'Release branch {repo.branch_name} does not exist in the repo {repo.repo_name}'
)
repo.change_repo_state(branch_name=repo.branch_name)
else:
repo.change_repo_state()
if repo.is_trigger:
git_commit_date = repo.get_time()
commit_timestamp = self.commit_time.timestamp() \
if self.commit_time \
else git_commit_date
for repo in self.repo_states:
if repo.commit_id == 'HEAD':
repo.prepare_repo()
if MediaSdkDirectories.is_release_branch(repo.branch_name):
if not repo.is_branch_exist(repo.branch_name):
raise BranchDoesNotExistException(
f'Release branch {repo.branch_name} does not exist in the repo {repo.repo_name}'
)
repo.change_repo_state(branch_name=repo.branch_name,
commit_time=commit_timestamp)
# if parameters '--commit-time', '--changed-repo' and '--repo-states' didn't set
# then variable 'commit_timestamp' is 'None' and 'HEAD' revisions be used
else:
repo.change_repo_state(commit_time=commit_timestamp)
def is_release_branch(raw_branch):
# TODO: need to unify with MediaSdkDirectories.is_release_branch method
"""
Checks if branch is release branch
Used as branch filter for pollers
"""
# Ignore pull request branches like 'refs/pull/*'
if raw_branch.startswith('refs/heads/'):
branch = raw_branch[11:]
if MediaSdkDirectories.is_release_branch(branch) or branch == 'master':
return True
return False
def is_limited_number_of_commits(pull_request, token=None):
"""
Checks if number of commits does not exceed the specified value
"""
github_commits_url = pull_request['commits_url']
data = get_data(github_commits_url,
additional_headers={'Authorization': f'token {token}'}
if token else None)
number_of_commits = len(data)
branch = pull_request['base']['ref']
if number_of_commits > MAX_NUM_COMMITS:
return False
if number_of_commits > MAX_NUM_COMMITS_RELEASE_BRANCH and \
(MediaSdkDirectories.is_release_branch(branch) or branch == 'master'):
return False
return True
def _check_branch(self):
"""
Check release branch
"""
self._log.info('Checking release branch')
if self._target_branch:
branch_to_check = self._target_branch
else:
branch_to_check = self._branch
if MediaSdkDirectories.is_release_branch(branch_to_check):
sdk_br, driver_br = convert_branch(branch_to_check)
for repo_name in self._release_repos:
if repo_name == 'media-driver':
self._release_branch[repo_name] = driver_br
else:
self._release_branch[repo_name] = sdk_br
"factory":
FACTORIES.init_package_factory,
"worker":
"ubuntu",
'triggers': [{
'builders': ['test-api-next', 'test'],
'filter':
GithubCommitFilter(
PRODUCTION_REPOS, lambda branch, target_branch:
(target_branch or branch) == 'master')
}, {
'builders': ['test'],
'filter':
GithubCommitFilter(
PRODUCTION_REPOS, lambda branch, target_branch:
MediaSdkDirectories.is_release_branch(target_branch or branch))
}]
}
}
FLOW = factories.Flow(BUILDERS, FACTORIES)
WORKERS = {
"windows": {
'b-1-26': {
"os": OsType.windows
},
'b-1-27': {
"os": OsType.windows
},
},
"mediasdk-api-next": {
"factory": FACTORIES.init_build_factory,
"product_conf_file": "conf_linux_public.py",
"product_type": Product_type.PUBLIC_LINUX_API_NEXT.value,
"build_type": Build_type.RELEASE.value,
"api_latest": True,
"fastboot": False,
"compiler": "gcc",
"compiler_version": "6.3.1",
"worker": "centos",
"dependency_name": 'mediasdk',
# Builder is enabled for not release branches
'triggers': [{'builders': ['libva'],
'filter': GithubCommitFilter(
PRODUCTION_REPOS,
lambda branch, target_branch: not MediaSdkDirectories.is_release_branch(
target_branch or branch))}]
},
"mediasdk-gcc-9.2.1": {
"factory": FACTORIES.init_build_factory,
"product_conf_file": "conf_linux_public.py",
"product_type": Product_type.PUBLIC_LINUX_GCC_LATEST.value,
"build_type": Build_type.RELEASE.value,
"api_latest": False,
"fastboot": False,
"compiler": "gcc",
"compiler_version": "9.2.1",
"worker": "ubuntu",
"dependency_name": 'mediasdk',
'triggers': [{'builders': ['libva'],
'filter': GithubCommitFilter(
def extract_repo(root_repo_dir,
repo_name,
branch,
commit_id=None,
commit_time=None,
proxy=False):
log = logging.getLogger('extract_repo.extract_repo')
try:
repo_url = MediaSdkDirectories.get_repo_url_by_name(repo_name)
if commit_id:
repo = git_worker.GitRepo(root_repo_dir=root_repo_dir,
repo_name=repo_name,
branch=branch,
url=repo_url,
commit_id=commit_id)
repo.prepare_repo()
repo.change_repo_state()
elif commit_time:
repo = git_worker.GitRepo(root_repo_dir=root_repo_dir,
repo_name=repo_name,
branch='master',
url=repo_url)
repo.prepare_repo()
if MediaSdkDirectories.is_release_branch(branch):
if not repo.is_branch_exist(branch):
raise git_worker.BranchDoesNotExistException(
f'Release branch {branch} does not exist in the repo {repo.repo_name}'
)
# repo.branch = branch
repo.change_repo_state(branch_name=branch,
commit_time=datetime.strptime(
commit_time,
'%Y-%m-%d %H:%M:%S').timestamp())
else:
repo.change_repo_state(commit_time=datetime.strptime(
commit_time, '%Y-%m-%d %H:%M:%S').timestamp())
else:
log.info(
'Commit id and timestamp not specified, clone HEAD of repository'
)
repo = git_worker.GitRepo(root_repo_dir=root_repo_dir,
repo_name=repo_name,
branch='master',
url=repo_url)
repo.prepare_repo()
if MediaSdkDirectories.is_release_branch(branch):
if not repo.is_branch_exist(branch):
raise git_worker.BranchDoesNotExistException(
f'Release branch {branch} does not exist in the repo {repo.repo_name}'
)
# repo.branch = branch
repo.change_repo_state(branch_name=branch)
else:
# repo.branch = master
repo.change_repo_state()
except Exception:
log.exception('Exception occurred')
exit_script(ErrorCode.CRITICAL)
def _extract(self):
"""
Get and prepare build repositories
Uses git_worker.py module
:return: None | Exception
"""
print('-' * 50)
self._log.info("EXTRACTING")
self._options['REPOS_DIR'].mkdir(parents=True, exist_ok=True)
self._options['PACK_DIR'].mkdir(parents=True, exist_ok=True)
triggered_repo = 'unknown'
if self._changed_repo:
repo_name, branch, commit_id = self._changed_repo.split(':')
triggered_repo = repo_name
if repo_name not in self._product_repos:
self._log.critical(
f'{repo_name} repository is not defined in the product configuration PRODUCT_REPOS'
)
return False
for repo, data in self._product_repos.items():
data['trigger'] = False
if repo == repo_name:
data['trigger'] = True
if self._target_branch:
data['target_branch'] = self._target_branch
if not data.get('branch'):
data['branch'] = branch
data['commit_id'] = commit_id
elif not data.get('branch'):
if self._target_branch and MediaSdkDirectories.is_release_branch(
self._target_branch):
data['branch'] = self._target_branch
elif MediaSdkDirectories.is_release_branch(
branch) and not self._target_branch:
data['branch'] = branch
elif self._repo_states:
for repo_name, values in self._repo_states.items():
if repo_name in self._product_repos:
if values['trigger']:
triggered_repo = repo_name
if values.get('target_branch'):
self._product_repos[repo_name][
'target_branch'] = values['target_branch']
self._product_repos[repo_name]['branch'] = values['branch']
self._product_repos[repo_name]['commit_id'] = values[
'commit_id']
self._product_repos[repo_name]['url'] = values['url']
self._product_repos[repo_name]['trigger'] = values[
'trigger']
elif self._manifest_file:
component = self._manifest.get_component(self._product)
for repo in component.repositories:
if repo.name in self._product_repos:
if repo.is_trigger:
triggered_repo = repo.name
if repo.target_branch:
self._product_repos[repo.name][
'target_branch'] = repo.target_branch
self._product_repos[repo.name]['branch'] = repo.branch
self._product_repos[repo.name]['commit_id'] = repo.revision
self._product_repos[repo.name]['url'] = repo.url
self._product_repos[repo.name]['trigger'] = repo.is_trigger
product_state = ProductState(self._product_repos,
self._options["REPOS_DIR"],
self._commit_time)
product_state.extract_all_repos()
product_state.save_repo_states(self._options["PACK_DIR"] /
'repo_states.json',
trigger=triggered_repo)
self._save_manifest(product_state)
shutil.copyfile(self._config_path,
self._options["PACK_DIR"] / self._config_path.name)
test_scenario = self._config_path.parent / f'{self._config_path.stem}_test{self._config_path.suffix}'
if test_scenario.exists():
shutil.copyfile(test_scenario,
self._options["PACK_DIR"] / test_scenario.name)
if not self._get_dependencies():
return False
if not self._run_build_config_actions(Stage.EXTRACT.value):
return False
return True
def _extract(self):
"""
Get and prepare build repositories
Uses git_worker.py module
:return: None | Exception
"""
print('-' * 50)
self.log.info("EXTRACTING")
self.options['REPOS_DIR'].mkdir(parents=True, exist_ok=True)
self.options['REPOS_FORKED_DIR'].mkdir(parents=True, exist_ok=True)
self.options['PACK_DIR'].mkdir(parents=True, exist_ok=True)
triggered_repo = 'unknown'
if self.changed_repo:
repo_name, branch, commit_id = self.changed_repo.split(':')
triggered_repo = repo_name
if repo_name not in self.product_repos:
self.log.critical(
f'{repo_name} repository is not defined in the product configuration PRODUCT_REPOS'
)
return False
for repo, data in self.product_repos.items():
if repo == repo_name:
if not data.get('branch'):
data['branch'] = branch
data['commit_id'] = commit_id
if self.repo_url:
data['url'] = self.repo_url
elif not data.get('branch'):
if MediaSdkDirectories.is_release_branch(branch):
data['branch'] = branch
elif self.repo_states:
for repo_name, values in self.repo_states.items():
if repo_name in self.product_repos:
if values['trigger']:
triggered_repo = repo_name
self.product_repos[repo_name]['branch'] = values['branch']
self.product_repos[repo_name]['commit_id'] = values[
'commit_id']
self.product_repos[repo_name]['url'] = values['url']
product_state = ProductState(self.product_repos,
self.options["REPOS_DIR"],
self.commit_time)
product_state.extract_all_repos()
product_state.save_repo_states(self.options["PACK_DIR"] /
'repo_states.json',
trigger=triggered_repo)
shutil.copyfile(self.build_config_path,
self.options["PACK_DIR"] / self.build_config_path.name)
if not self._run_build_config_actions(Stage.EXTRACT.value):
return False
return True
'builders': ['build-libva']}]
},
"build-api-next": {
"factory": FACTORIES.init_build_factory,
"product_conf_file": "conf_linux_public.py",
"product_type": Product_type.PUBLIC_LINUX_API_NEXT.value,
"build_type": Build_type.RELEASE.value,
"api_latest": True,
"fastboot": False,
"compiler": "gcc",
"compiler_version": "6.3.1",
"worker": "centos",
# Builder is enabled for not release branches
'triggers': [{'repositories': PRODUCTION_REPOS,
'branches': lambda branch: not MediaSdkDirectories.is_release_branch(branch),
'builders': ['build-libva']}]
},
"build-gcc-8.2.0": {
"factory": FACTORIES.init_build_factory,
"product_conf_file": "conf_linux_public.py",
"product_type": Product_type.PUBLIC_LINUX_GCC_LATEST.value,
"build_type": Build_type.RELEASE.value,
"api_latest": False,
"fastboot": False,
"compiler": "gcc",
"compiler_version": "8.2.0",
"worker": "ubuntu",
'triggers': [{'repositories': PRODUCTION_REPOS,
'branches': lambda branch: not MediaSdkDirectories.is_release_branch(branch),