def extract_all_repos(self): """ Get repositories and checkout them to the right state :return: None """ git_commit_date = None for repo in self.repo_states: if repo.commit_id != 'HEAD': repo.prepare_repo() if MediaSdkDirectories.is_release_branch(repo.branch_name): if not repo.is_branch_exist(repo.branch_name): raise BranchDoesNotExistException( f'Release branch {repo.branch_name} does not exist in the repo {repo.repo_name}' ) repo.change_repo_state(branch_name=repo.branch_name) else: repo.change_repo_state() if repo.is_trigger: git_commit_date = repo.get_time() commit_timestamp = self.commit_time.timestamp() \ if self.commit_time \ else git_commit_date for repo in self.repo_states: if repo.commit_id == 'HEAD': repo.prepare_repo() if MediaSdkDirectories.is_release_branch(repo.branch_name): if not repo.is_branch_exist(repo.branch_name): raise BranchDoesNotExistException( f'Release branch {repo.branch_name} does not exist in the repo {repo.repo_name}' ) repo.change_repo_state(branch_name=repo.branch_name, commit_time=commit_timestamp) # if parameters '--commit-time', '--changed-repo' and '--repo-states' didn't set # then variable 'commit_timestamp' is 'None' and 'HEAD' revisions be used else: repo.change_repo_state(commit_time=commit_timestamp)
def extract_repo(root_dir, repo_name, url=None, branch='master', commit_id='HEAD', commit_time=None, proxy=False): """ Prepare repository :param root_dir: Directory where repositories will clone :param repo_name: Name of repository :param url: Url to repository :param branch: Branch of repository :param commit_id: Revision of commit :param commit_time: Time for getting slice of commits of repositories :param proxy: Proxy enabling :return: None | Exception """ log = logging.getLogger('ProductState.extract_repo') if url is None: url = MediaSdkDirectories.get_repo_url_by_name(repo_name) root_dir = pathlib.Path(root_dir) root_dir.mkdir(parents=True, exist_ok=True) if commit_time and not isinstance(commit_time, float): commit_time = datetime.strptime(commit_time, '%Y-%m-%d %H:%M:%S').timestamp() if not commit_time and not commit_id: log.info( 'Commit id and timestamp not specified, clone HEAD of repository') try: repo = GitRepo(root_repo_dir=root_dir, repo_name=repo_name, branch=branch, url=url, commit_id=commit_id) repo.prepare_repo() repo.change_repo_state(commit_time=commit_time) except Exception: log.exception('Exception occurred') log.info("EXTRACTING FAILED") exit(ErrorCode.CRITICAL.value) log.info("EXTRACTING COMPLETED")
def is_release_branch(raw_branch): # TODO: need to unify with MediaSdkDirectories.is_release_branch method """ Checks if branch is release branch Used as branch filter for pollers """ # Ignore pull request branches like 'refs/pull/*' if raw_branch.startswith('refs/heads/'): branch = raw_branch[11:] if MediaSdkDirectories.is_release_branch(branch) or branch == 'master': return True return False
def _get_dependencies(self): deps = self._config_variables.get("DEPENDENCIES", {}) if not deps: return True try: deps_dir = self._options['DEPENDENCIES_DIR'] self._log.info( f'Dependencies was found. Trying to extract to {deps_dir}') deps_dir.mkdir(parents=True, exist_ok=True) self._log.info(f'Creating manifest') for dependency in deps: self._log.info(f'Getting component {dependency}') comp = self._manifest.get_component(dependency) if comp: trigger_repo = comp.trigger_repository if trigger_repo: dep_dir = MediaSdkDirectories.get_build_dir( trigger_repo.target_branch if trigger_repo.target_branch else trigger_repo.branch, Build_event.COMMIT.value, trigger_repo.revision, comp.product_type, Build_type.RELEASE.value, product=dependency) try: self._log.info( f'Extracting {dependency} {comp.product_type} artifacts' ) # TODO: Extension hardcoded for open source. Need to use only .zip in future. extract_archive(dep_dir / f'install_pkg.tar.gz', deps_dir / dependency) except Exception: self._log.exception('Can not extract archive') return False else: self._log.error('There is no repository as a trigger') return False else: self._log.error( f'Component {dependency} does not exist in manifest') return False except Exception: self._log.exception('Exception occurred:') return False return True
def get_manifest_path(self, props): """ Return path to main manifest on share as string for target worker. get_path and util.Interpolate wrappers are not needed. """ return str( MediaSdkDirectories.get_commit_dir( props.getProperty('target_branch') or props.getProperty('branch'), props.getProperty('event_type'), props.getProperty("revision"), os_type=props.getProperty('os'), # TODO: import the const from common product='manifest') / 'manifest.yml')
def is_limited_number_of_commits(pull_request, token=None): """ Checks if number of commits does not exceed the specified value """ github_commits_url = pull_request['commits_url'] data = get_data(github_commits_url, additional_headers={'Authorization': f'token {token}'} if token else None) number_of_commits = len(data) branch = pull_request['base']['ref'] if number_of_commits > MAX_NUM_COMMITS: return False if number_of_commits > MAX_NUM_COMMITS_RELEASE_BRANCH and \ (MediaSdkDirectories.is_release_branch(branch) or branch == 'master'): return False return True
def extract_open_source_infrastructure(root_dir, branch, commit_id, commit_time, manifest): log = logging.getLogger('extract_repo.extract_open_source_infrastructure') repos = MediaSdkDirectories() open_source_product_configs_repo = repos.open_source_product_configs_repo open_source_infra_repo = repos.open_source_infrastructure_repo # Extract product configs if not manifest: extract_repo(root_repo_dir=root_dir, repo_name=open_source_product_configs_repo, branch=branch, commit_id=commit_id, commit_time=commit_time) manifest_data = Manifest(root_dir / open_source_product_configs_repo / 'manifest.yml') else: manifest_data = Manifest(manifest) product_conf = manifest_data.get_component('infra').get_repository( open_source_product_configs_repo) extract_repo(root_repo_dir=root_dir, repo_name=product_conf.name, branch=product_conf.branch, commit_id=product_conf.revision, commit_time=commit_time) open_source_infra = manifest_data.get_component('infra').get_repository( open_source_infra_repo) # Extract open source infrastructure extract_repo(root_repo_dir=root_dir, repo_name=open_source_infra.name, branch=open_source_infra.branch, commit_id=open_source_infra.revision) try: log.info(f"Copy secrets") shutil.copyfile( str(pathlib.Path('msdk_secrets.py').absolute()), str(root_dir / open_source_infra_repo / 'common' / 'msdk_secrets.py')) except Exception: log.exception('Can not create infrastructure package') # TODO: An exit from script should be in main() exit_script(ErrorCode.CRITICAL)
def generate_build_config(self): """ Build configuration file parser :return: None | Exception """ global_vars = { 'action': self._action, 'vs_component': self._vs_component, 'options': self.options, 'stage': Stage, 'copy_win_files': copy_win_files, 'args': self.custom_cli_args, 'log': self.log, 'product_type': self.product_type, 'build_event': self.build_event, # TODO should be in lower case 'DEV_PKG_DATA_TO_ARCHIVE': self.dev_pkg_data_to_archive, 'INSTALL_PKG_DATA_TO_ARCHIVE': self.install_pkg_data_to_archive, 'get_build_number': get_build_number, 'get_api_version': self._get_api_version, 'branch_name': self.branch_name, 'update_config': self._update_config, 'target_arch': self.target_arch, 'commit_time': self.commit_time } exec( open(self.build_config_path).read(), global_vars, self.config_variables) # TODO add product_repos to global_vars if 'PRODUCT_REPOS' in self.config_variables: for repo in self.config_variables['PRODUCT_REPOS']: self.product_repos[repo['name']] = { 'branch': repo.get('branch'), 'commit_id': repo.get('commit_id'), 'url': MediaSdkDirectories.get_repo_url_by_name(repo['name']) } return True
def check_component_existence(path_to_manifest, component_name): log = logging.getLogger('component_checker') log.info(f"Getting data for {component_name} from {path_to_manifest}") manifest = Manifest(pathlib.Path(path_to_manifest)) component = manifest.get_component(component_name) repository = component.get_repository(component_name) component_dir = MediaSdkDirectories.get_build_dir( repository.target_branch if repository.target_branch else repository.branch, Build_event.COMMIT.value, repository.revision, component.product_type, Build_type.RELEASE.value, product=component_name) if component_dir.exists(): log.info(f"Directory {component_dir} exists") # This is stop phrase for buildbot to skip all build stages log.info(SKIP_BUILDING_DEPENDENCY_PHRASE) else: log.info(f"Directory {component_dir} doesn't exist")
def _check_branch(self): """ Check release branch """ self._log.info('Checking release branch') if self._target_branch: branch_to_check = self._target_branch else: branch_to_check = self._branch if MediaSdkDirectories.is_release_branch(branch_to_check): sdk_br, driver_br = convert_branch(branch_to_check) for repo_name in self._release_repos: if repo_name == 'media-driver': self._release_branch[repo_name] = driver_br else: self._release_branch[repo_name] = sdk_br
"mediasdk-api-next": { "factory": FACTORIES.init_build_factory, "product_conf_file": "conf_linux_public.py", "product_type": Product_type.PUBLIC_LINUX_API_NEXT.value, "build_type": Build_type.RELEASE.value, "api_latest": True, "fastboot": False, "compiler": "gcc", "compiler_version": "6.3.1", "worker": "centos", "dependency_name": 'mediasdk', # Builder is enabled for not release branches 'triggers': [{'builders': ['libva'], 'filter': GithubCommitFilter( PRODUCTION_REPOS, lambda branch, target_branch: not MediaSdkDirectories.is_release_branch( target_branch or branch))}] }, "mediasdk-gcc-9.2.1": { "factory": FACTORIES.init_build_factory, "product_conf_file": "conf_linux_public.py", "product_type": Product_type.PUBLIC_LINUX_GCC_LATEST.value, "build_type": Build_type.RELEASE.value, "api_latest": False, "fastboot": False, "compiler": "gcc", "compiler_version": "9.2.1", "worker": "ubuntu", "dependency_name": 'mediasdk', 'triggers': [{'builders': ['libva'], 'filter': GithubCommitFilter(
def extract_repo(root_repo_dir, repo_name, branch, commit_id=None, commit_time=None, proxy=False): log = logging.getLogger('extract_repo.extract_repo') try: repo_url = MediaSdkDirectories.get_repo_url_by_name(repo_name) if commit_id: repo = git_worker.GitRepo(root_repo_dir=root_repo_dir, repo_name=repo_name, branch=branch, url=repo_url, commit_id=commit_id) repo.prepare_repo() repo.change_repo_state() elif commit_time: repo = git_worker.GitRepo(root_repo_dir=root_repo_dir, repo_name=repo_name, branch='master', url=repo_url) repo.prepare_repo() if MediaSdkDirectories.is_release_branch(branch): if not repo.is_branch_exist(branch): raise git_worker.BranchDoesNotExistException( f'Release branch {branch} does not exist in the repo {repo.repo_name}' ) # repo.branch = branch repo.change_repo_state(branch_name=branch, commit_time=datetime.strptime( commit_time, '%Y-%m-%d %H:%M:%S').timestamp()) else: repo.change_repo_state(commit_time=datetime.strptime( commit_time, '%Y-%m-%d %H:%M:%S').timestamp()) else: log.info( 'Commit id and timestamp not specified, clone HEAD of repository' ) repo = git_worker.GitRepo(root_repo_dir=root_repo_dir, repo_name=repo_name, branch='master', url=repo_url) repo.prepare_repo() if MediaSdkDirectories.is_release_branch(branch): if not repo.is_branch_exist(branch): raise git_worker.BranchDoesNotExistException( f'Release branch {branch} does not exist in the repo {repo.repo_name}' ) # repo.branch = branch repo.change_repo_state(branch_name=branch) else: # repo.branch = master repo.change_repo_state() except Exception: log.exception('Exception occurred') exit_script(ErrorCode.CRITICAL)
def extract_private_infrastructure(root_dir, branch, commit_id, commit_time): log = logging.getLogger('extract_repo.extract_private_infrastructure') infrastructure_root_dir = root_dir / 'infrastructure' # We save and update repos in temporary folder and create infrastructure package from it # So, not needed extracting repo to the beginning each time original_repos_dir = root_dir / 'tmp_infrastructure' repos = MediaSdkDirectories() open_source_product_configs_repo = repos.open_source_product_configs_repo open_source_infra_repo = repos.open_source_infrastructure_repo closed_source_product_configs_repo = repos.closed_source_product_configs_repo closed_source_infra_repo = repos.closed_source_infrastructure_repo # Extract open source infrastructure and product configs extract_open_source_infrastructure(original_repos_dir, branch, commit_id, commit_time) # Extract closed source product configs extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_product_configs_repo, branch='master', commit_time=commit_time) # Get revision of closed source infrastructure from closed source product configs repo configs_dir = original_repos_dir / closed_source_product_configs_repo sys.path.append(str(configs_dir)) import infrastructure_version infrastructure_version = reload(infrastructure_version) closed_source_infra_version = infrastructure_version.CLOSED_SOURCE # Extract closed source infrastructure extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_infra_repo, branch=closed_source_infra_version['branch'], commit_id=closed_source_infra_version['commit_id']) log.info('-' * 50) log.info(f"Create infrastructure package") try: log.info(f"- Delete existing infrastructure") if infrastructure_root_dir.exists(): remove_directory(str(infrastructure_root_dir)) log.info(f"- Copy open source infrastructure") copy_tree(str(original_repos_dir / open_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Copy closed source infrastructure") copy_tree(str(original_repos_dir / closed_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Remove closed source static data") (infrastructure_root_dir / 'common' / 'static_closed_data.py').unlink() log.info(f"- Copy open source product configs") copy_tree( str(original_repos_dir / open_source_product_configs_repo), str(infrastructure_root_dir / open_source_product_configs_repo)) log.info(f"- Copy closed source product configs") copy_tree( str(original_repos_dir / closed_source_product_configs_repo), str(infrastructure_root_dir / open_source_product_configs_repo)) # log.info(f"Copy secrets") shutil.copyfile( str(pathlib.Path('msdk_secrets.py').absolute()), str(infrastructure_root_dir / 'common' / 'msdk_secrets.py')) except Exception: log.exception('Can not create infrastructure package') exit_script(ErrorCode.CRITICAL)
def extract_private_infrastructure(root_dir, branch, commit_id, commit_time, manifest): log = logging.getLogger('extract_repo.extract_private_infrastructure') infrastructure_root_dir = root_dir / 'infrastructure' configs_root_dir = root_dir / 'product-configs' # We save and update repos in temporary folder and create infrastructure package from it # So, not needed extracting repo to the beginning each time original_repos_dir = root_dir / 'tmp_infrastructure' repos = MediaSdkDirectories() open_source_product_configs_repo = repos.open_source_product_configs_repo open_source_infra_repo = repos.open_source_infrastructure_repo closed_source_product_configs_repo = repos.closed_source_product_configs_repo closed_source_infra_repo = repos.closed_source_infrastructure_repo # Extract open source infrastructure and product configs extract_open_source_infrastructure(original_repos_dir, branch, commit_id, commit_time, manifest) # Extract closed source product configs extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_product_configs_repo, branch='master', commit_time=commit_time) manifest_path = original_repos_dir / closed_source_product_configs_repo / 'manifest.yml' manifest_data = Manifest(manifest_path) closed_source_infra = manifest_data.get_component('infra').get_repository( closed_source_infra_repo) # Extract closed source infrastructure extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_infra.name, branch=closed_source_infra.branch, commit_id=closed_source_infra.revision) # Event repository in infra component for private builds is product-configs, # so need to change default trigger value manifest_data.get_component('infra').build_info.set_trigger( open_source_product_configs_repo) manifest_data.save_manifest(manifest_path) log.info('-' * 50) log.info(f"Create infrastructure package") try: log.info(f"- Delete existing infrastructure") if infrastructure_root_dir.exists(): remove_directory(str(infrastructure_root_dir)) if configs_root_dir.exists(): remove_directory(str(configs_root_dir)) log.info(f"- Copy open source infrastructure") copy_tree(str(original_repos_dir / open_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Copy closed source infrastructure") copy_tree(str(original_repos_dir / closed_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Remove closed source static data") (infrastructure_root_dir / 'common' / 'static_closed_data.py').unlink() log.info(f"- Copy open source product configs") copy_tree(str(original_repos_dir / open_source_product_configs_repo), str(configs_root_dir)) log.info(f"- Copy closed source product configs") copy_tree(str(original_repos_dir / closed_source_product_configs_repo), str(configs_root_dir)) # log.info(f"Copy secrets") shutil.copyfile( str(pathlib.Path('msdk_secrets.py').absolute()), str(infrastructure_root_dir / 'common' / 'msdk_secrets.py')) except Exception: log.exception('Can not create infrastructure package') exit_script(ErrorCode.CRITICAL)
def extract_closed_source_infrastructure(root_dir, branch, commit_id, commit_time, manifest): log = logging.getLogger( 'extract_repo.extract_closed_source_infrastructure') infrastructure_root_dir = root_dir / 'infrastructure' configs_root_dir = root_dir / 'product-configs' # We save and update repos in temporary folder and create infrastructure package from it # So, not needed extracting repo to the beginning each time original_repos_dir = root_dir / 'tmp_infrastructure' repos = MediaSdkDirectories() closed_source_product_configs_repo = repos.closed_source_product_configs_repo open_source_infra_repo = repos.open_source_infrastructure_repo closed_source_infra_repo = repos.closed_source_infrastructure_repo # Extract product configs if not manifest: extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_product_configs_repo, branch=branch, commit_id=commit_id, commit_time=commit_time) manifest_data = Manifest(original_repos_dir / closed_source_product_configs_repo / 'manifest.yml') else: manifest_data = Manifest(manifest) product_conf = manifest_data.get_component('infra').get_repository( closed_source_product_configs_repo) extract_repo(root_repo_dir=original_repos_dir, repo_name=product_conf.name, branch=product_conf.branch, commit_id=product_conf.revision, commit_time=commit_time) open_source_infra = manifest_data.get_component('infra').get_repository( open_source_infra_repo) closed_source_infra = manifest_data.get_component('infra').get_repository( closed_source_infra_repo) # Extract open source infrastructure # Set proxy for access to GitHub extract_repo(root_repo_dir=original_repos_dir, repo_name=open_source_infra.name, branch=open_source_infra.branch, commit_id=open_source_infra.revision, proxy=True) # Extract closed source part of infrastructure extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_infra.name, branch=closed_source_infra.branch, commit_id=closed_source_infra.revision) log.info('-' * 50) log.info(f"Create infrastructure package") try: log.info(f"- Delete existing infrastructure") if infrastructure_root_dir.exists(): remove_directory(str(infrastructure_root_dir)) if configs_root_dir.exists(): remove_directory(str(configs_root_dir)) log.info(f"- Copy open source infrastructure") copy_tree(str(original_repos_dir / open_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Copy closed source infrastructure") copy_tree(str(original_repos_dir / closed_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Copy product configs") copy_tree(str(original_repos_dir / closed_source_product_configs_repo), str(configs_root_dir)) # log.info(f"Copy secrets") shutil.copyfile( str(pathlib.Path('msdk_secrets.py').absolute()), str(infrastructure_root_dir / 'common' / 'msdk_secrets.py')) except Exception: log.exception('Can not create infrastructure package') exit_script(ErrorCode.CRITICAL)
def main(): """ Tests runner :return: None """ parser = argparse.ArgumentParser( prog="test_adapter.py", formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("--version", action="version", version="%(prog)s 1.0") parser.add_argument('-br', "--branch", metavar="String", required=True, help="Branch of triggered repository") parser.add_argument( '-e', "--build-event", default='commit', choices=[build_event.value for build_event in Build_event], help='Event of commit') parser.add_argument('-c', "--commit-id", metavar="String", required=True, help="SHA of triggered commit") parser.add_argument( '-p', "--product-type", default='closed_linux', choices=[product_type.value for product_type in Product_type], help='Type of product') parser.add_argument( '-b', "--build-type", default='release', choices=[build_type.value for build_type in Build_type], help='Type of build') parser.add_argument('-d', "--root-dir", metavar="PATH", required=True, help="Path to worker directory") args = parser.parse_args() directories_layout = { 'branch': args.branch, 'build_event': args.build_event, 'commit_id': args.commit_id, 'product_type': args.product_type, 'build_type': args.build_type, } build_artifacts_dir = MediaSdkDirectories.get_build_dir( **directories_layout) tests_artifacts_dir = MediaSdkDirectories.get_test_dir( **directories_layout) tests_artifacts_url = MediaSdkDirectories.get_test_url( **directories_layout) log = logging.getLogger('test_adapter.log') adapter = TedAdapter(build_artifacts_dir, tests_artifacts_dir, tests_artifacts_url, root_dir=pathlib.Path(args.root_dir)) # Install third parties for msdk if not adapter.install_pkgs(THIRD_PARTY): log.info(f'Required packages "{THIRD_PARTY}" were not installed\n') exit(TestReturnCodes.INFRASTRUCTURE_ERROR.value) # Check existence of driver check_driver() # Install msdk if not adapter.install_pkgs(['mediasdk'], clean_dir=True): log.info(f'Package "mediasdk" was not installed\n') exit(TestReturnCodes.INFRASTRUCTURE_ERROR.value) try: tests_return_code = adapter.run_test() except Exception: print("Exception occurred:\n", traceback.format_exc()) # TODO return json string tests_return_code = TestReturnCodes.INFRASTRUCTURE_ERROR.value try: tests_return_code |= adapter.run_fei_tests() except Exception: print("Exception occurred:\n", traceback.format_exc()) # TODO return json string tests_return_code |= TestReturnCodes.INFRASTRUCTURE_ERROR.value try: adapter.copy_logs_to_share() except Exception: print("Exception occurred while copying results:\n", traceback.format_exc()) tests_return_code |= TestReturnCodes.INFRASTRUCTURE_ERROR.value exit(tests_return_code)
def _copy(self): """ Copy 'pack' stage results to share folder :return: None | Exception """ print('-' * 50) self._log.info("COPYING") branch = 'unknown' commit_id = 'unknown' if self._changed_repo: repo_name, branch, commit_id = self._changed_repo.split(':') if self._target_branch: branch = self._target_branch if commit_id == 'HEAD': commit_id = ProductState.get_head_revision( self._options['REPOS_DIR'] / repo_name) elif self._repo_states: for repo in self._repo_states.values(): if repo['trigger']: branch = repo['target_branch'] if repo.get( 'target_branch') else repo['branch'] commit_id = repo['commit_id'] elif self._manifest_file: component = self._manifest.get_component(self._product) repo = component.trigger_repository branch = repo.target_branch if repo.target_branch else repo.branch commit_id = repo.revision else: # "--changed-repo" or "--repo-states" arguments are not set, "HEAD" revision and "master" branch are used branch = 'master' commit_id = 'HEAD' build_dir = MediaSdkDirectories.get_build_dir( branch, self._build_event, commit_id, self._product_type, self._options["BUILD_TYPE"], product=self._product) build_url = MediaSdkDirectories.get_build_url( branch, self._build_event, commit_id, self._product_type, self._options["BUILD_TYPE"], product=self._product) build_root_dir = MediaSdkDirectories.get_root_builds_dir() rotate_dir(build_dir) self._log.info('Copy to %s', build_dir) self._log.info('Artifacts are available by: %s', build_url) # Workaround for copying to samba share on Linux # to avoid exceptions while setting Linux permissions. _orig_copystat = shutil.copystat shutil.copystat = lambda x, y, follow_symlinks=True: x shutil.copytree(self._options['PACK_DIR'], build_dir) shutil.copystat = _orig_copystat if not self._run_build_config_actions(Stage.COPY.value): return False if self._build_state_file.exists(): with self._build_state_file.open() as state: build_state = json.load(state) if build_state['status'] == "PASS": last_build_path = build_dir.relative_to(build_root_dir) last_build_file = build_dir.parent.parent / f'last_build_{self._product_type}' last_build_file.write_text(str(last_build_path)) return True
def _extract(self): """ Get and prepare build repositories Uses git_worker.py module :return: None | Exception """ print('-' * 50) self._log.info("EXTRACTING") self._options['REPOS_DIR'].mkdir(parents=True, exist_ok=True) self._options['PACK_DIR'].mkdir(parents=True, exist_ok=True) triggered_repo = 'unknown' if self._changed_repo: repo_name, branch, commit_id = self._changed_repo.split(':') triggered_repo = repo_name if repo_name not in self._product_repos: self._log.critical( f'{repo_name} repository is not defined in the product configuration PRODUCT_REPOS' ) return False for repo, data in self._product_repos.items(): data['trigger'] = False if repo == repo_name: data['trigger'] = True if self._target_branch: data['target_branch'] = self._target_branch if not data.get('branch'): data['branch'] = branch data['commit_id'] = commit_id elif not data.get('branch'): if self._target_branch and MediaSdkDirectories.is_release_branch( self._target_branch): data['branch'] = self._target_branch elif MediaSdkDirectories.is_release_branch( branch) and not self._target_branch: data['branch'] = branch elif self._repo_states: for repo_name, values in self._repo_states.items(): if repo_name in self._product_repos: if values['trigger']: triggered_repo = repo_name if values.get('target_branch'): self._product_repos[repo_name][ 'target_branch'] = values['target_branch'] self._product_repos[repo_name]['branch'] = values['branch'] self._product_repos[repo_name]['commit_id'] = values[ 'commit_id'] self._product_repos[repo_name]['url'] = values['url'] self._product_repos[repo_name]['trigger'] = values[ 'trigger'] elif self._manifest_file: component = self._manifest.get_component(self._product) for repo in component.repositories: if repo.name in self._product_repos: if repo.is_trigger: triggered_repo = repo.name if repo.target_branch: self._product_repos[repo.name][ 'target_branch'] = repo.target_branch self._product_repos[repo.name]['branch'] = repo.branch self._product_repos[repo.name]['commit_id'] = repo.revision self._product_repos[repo.name]['url'] = repo.url self._product_repos[repo.name]['trigger'] = repo.is_trigger product_state = ProductState(self._product_repos, self._options["REPOS_DIR"], self._commit_time) product_state.extract_all_repos() product_state.save_repo_states(self._options["PACK_DIR"] / 'repo_states.json', trigger=triggered_repo) self._save_manifest(product_state) shutil.copyfile(self._config_path, self._options["PACK_DIR"] / self._config_path.name) test_scenario = self._config_path.parent / f'{self._config_path.stem}_test{self._config_path.suffix}' if test_scenario.exists(): shutil.copyfile(test_scenario, self._options["PACK_DIR"] / test_scenario.name) if not self._get_dependencies(): return False if not self._run_build_config_actions(Stage.EXTRACT.value): return False return True
def _extract(self): """ Get and prepare build repositories Uses git_worker.py module :return: None | Exception """ print('-' * 50) self.log.info("EXTRACTING") self.options['REPOS_DIR'].mkdir(parents=True, exist_ok=True) self.options['REPOS_FORKED_DIR'].mkdir(parents=True, exist_ok=True) self.options['PACK_DIR'].mkdir(parents=True, exist_ok=True) triggered_repo = 'unknown' if self.changed_repo: repo_name, branch, commit_id = self.changed_repo.split(':') triggered_repo = repo_name if repo_name not in self.product_repos: self.log.critical( f'{repo_name} repository is not defined in the product configuration PRODUCT_REPOS' ) return False for repo, data in self.product_repos.items(): if repo == repo_name: if not data.get('branch'): data['branch'] = branch data['commit_id'] = commit_id if self.repo_url: data['url'] = self.repo_url elif not data.get('branch'): if MediaSdkDirectories.is_release_branch(branch): data['branch'] = branch elif self.repo_states: for repo_name, values in self.repo_states.items(): if repo_name in self.product_repos: if values['trigger']: triggered_repo = repo_name self.product_repos[repo_name]['branch'] = values['branch'] self.product_repos[repo_name]['commit_id'] = values[ 'commit_id'] self.product_repos[repo_name]['url'] = values['url'] product_state = ProductState(self.product_repos, self.options["REPOS_DIR"], self.commit_time) product_state.extract_all_repos() product_state.save_repo_states(self.options["PACK_DIR"] / 'repo_states.json', trigger=triggered_repo) shutil.copyfile(self.build_config_path, self.options["PACK_DIR"] / self.build_config_path.name) if not self._run_build_config_actions(Stage.EXTRACT.value): return False return True
"factory": FACTORIES.init_package_factory, "worker": "ubuntu", 'triggers': [{ 'builders': ['test-api-next', 'test'], 'filter': GithubCommitFilter( PRODUCTION_REPOS, lambda branch, target_branch: (target_branch or branch) == 'master') }, { 'builders': ['test'], 'filter': GithubCommitFilter( PRODUCTION_REPOS, lambda branch, target_branch: MediaSdkDirectories.is_release_branch(target_branch or branch)) }] } } FLOW = factories.Flow(BUILDERS, FACTORIES) WORKERS = { "windows": { 'b-1-26': { "os": OsType.windows }, 'b-1-27': { "os": OsType.windows }, },
'builders': ['build-libva']}] }, "build-api-next": { "factory": FACTORIES.init_build_factory, "product_conf_file": "conf_linux_public.py", "product_type": Product_type.PUBLIC_LINUX_API_NEXT.value, "build_type": Build_type.RELEASE.value, "api_latest": True, "fastboot": False, "compiler": "gcc", "compiler_version": "6.3.1", "worker": "centos", # Builder is enabled for not release branches 'triggers': [{'repositories': PRODUCTION_REPOS, 'branches': lambda branch: not MediaSdkDirectories.is_release_branch(branch), 'builders': ['build-libva']}] }, "build-gcc-8.2.0": { "factory": FACTORIES.init_build_factory, "product_conf_file": "conf_linux_public.py", "product_type": Product_type.PUBLIC_LINUX_GCC_LATEST.value, "build_type": Build_type.RELEASE.value, "api_latest": False, "fastboot": False, "compiler": "gcc", "compiler_version": "8.2.0", "worker": "ubuntu", 'triggers': [{'repositories': PRODUCTION_REPOS, 'branches': lambda branch: not MediaSdkDirectories.is_release_branch(branch),