def _clean(self): """ Clean temporary directory """ self._log.info(f'Cleaning {self._tmp_dir}') try: if self._tmp_dir.exists(): remove_directory(str(self._tmp_dir)) self._log.info(f"Removed old repository in {self._tmp_dir}") self._tmp_dir.mkdir(exist_ok=True) except Exception as e: self._log.exception(f'Failed to clean {self._tmp_dir}: %s', e) return False return True
def clone(self): """ Clone repo :return: None """ # checking correctness of git repository # if dir is not repository, it will be removed if self.local_repo_dir.exists(): try: git.Repo(str(self.local_repo_dir)) except git.InvalidGitRepositoryError: self.log.info('Remove broken repo %s', self.local_repo_dir) remove_directory(self.local_repo_dir) if not self.local_repo_dir.exists(): self.log.info("Clone repo " + self.repo_name) git.Git().clone(self.url, str(self.local_repo_dir))
def extract_private_infrastructure(root_dir, branch, commit_id, commit_time): log = logging.getLogger('extract_repo.extract_private_infrastructure') infrastructure_root_dir = root_dir / 'infrastructure' # We save and update repos in temporary folder and create infrastructure package from it # So, not needed extracting repo to the beginning each time original_repos_dir = root_dir / 'tmp_infrastructure' repos = MediaSdkDirectories() open_source_product_configs_repo = repos.open_source_product_configs_repo open_source_infra_repo = repos.open_source_infrastructure_repo closed_source_product_configs_repo = repos.closed_source_product_configs_repo closed_source_infra_repo = repos.closed_source_infrastructure_repo # Extract open source infrastructure and product configs extract_open_source_infrastructure(original_repos_dir, branch, commit_id, commit_time) # Extract closed source product configs extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_product_configs_repo, branch='master', commit_time=commit_time) # Get revision of closed source infrastructure from closed source product configs repo configs_dir = original_repos_dir / closed_source_product_configs_repo sys.path.append(str(configs_dir)) import infrastructure_version infrastructure_version = reload(infrastructure_version) closed_source_infra_version = infrastructure_version.CLOSED_SOURCE # Extract closed source infrastructure extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_infra_repo, branch=closed_source_infra_version['branch'], commit_id=closed_source_infra_version['commit_id']) log.info('-' * 50) log.info(f"Create infrastructure package") try: log.info(f"- Delete existing infrastructure") if infrastructure_root_dir.exists(): remove_directory(str(infrastructure_root_dir)) log.info(f"- Copy open source infrastructure") copy_tree(str(original_repos_dir / open_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Copy closed source infrastructure") copy_tree(str(original_repos_dir / closed_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Remove closed source static data") (infrastructure_root_dir / 'common' / 'static_closed_data.py').unlink() log.info(f"- Copy open source product configs") copy_tree( str(original_repos_dir / open_source_product_configs_repo), str(infrastructure_root_dir / open_source_product_configs_repo)) log.info(f"- Copy closed source product configs") copy_tree( str(original_repos_dir / closed_source_product_configs_repo), str(infrastructure_root_dir / open_source_product_configs_repo)) # log.info(f"Copy secrets") shutil.copyfile( str(pathlib.Path('msdk_secrets.py').absolute()), str(infrastructure_root_dir / 'common' / 'msdk_secrets.py')) except Exception: log.exception('Can not create infrastructure package') exit_script(ErrorCode.CRITICAL)
def increase_build_number(local_repo_path, component, branch): """ Increase build number by 1 in remote repository, if it is the same for local and remote repositories This condition is needed to avoid increasing build number while rebuilds Function extracts product-configs repo in following layout to push change to remote repository ../tmp/product-configs ../origin_repo_path :param local_repo_path: path to local repository with "build_numbers.json" file :type local_repo_path: String | pathlib.Path :param component: Component name. Need for finding certain build number :type component: String :param branch: Name of branch. Need for finding certain build number :type branch: String """ log = logging.getLogger('build_number.increase_build_number') log.info(f'Increasing build number in {branch} branch for {component}') build_numbers_file = 'build_numbers.json' log.info(f'Get build number from local repository') current_build_number = get_build_number( repo_path=pathlib.Path(local_repo_path), component=component, branch=branch) if current_build_number == 0: log.error( f'Local build number must not be 0\n' f'Check that {pathlib.Path(local_repo_path) / build_numbers_file} contains appropriate {branch} branch for {component}' ) return False repo_name = pathlib.Path(local_repo_path).name temp_dir = (pathlib.Path(local_repo_path) / '..' / 'tmp').resolve() latest_version_repo_path = temp_dir / repo_name latest_build_number_path = latest_version_repo_path / build_numbers_file if temp_dir.exists(): log.info(f"Remove old repository in {temp_dir}") remove_directory(str(temp_dir)) temp_dir.mkdir(exist_ok=True) log.warning(f'Redefine {branch} to "master"') # TODO: use extract_repo from git_worker in one_ci_dev branch branch = 'master' extract_repo(root_repo_dir=temp_dir, repo_name=repo_name, branch=branch, commit_id='HEAD') log.info( f'Getting build number from HEAD of {branch} branch for repo in {latest_version_repo_path}' ) latest_git_build_number = get_build_number( repo_path=latest_version_repo_path, component=component, branch=branch) if current_build_number != latest_git_build_number: log.warning( f'Build numbers in remote ({latest_git_build_number}) and local ({current_build_number}) repositories are not equal\n' f'It maybe because this is rebuild of old build for which build number already has been increased\n' f'Stop operation') return False log.info('Increasing build number') if latest_build_number_path.exists(): try: log.info(f'\tChanging build numbers file') with latest_build_number_path.open('r+') as build_number_file: build_numbers = json.load(build_number_file) new_build_number = build_numbers[component][branch] + 1 build_numbers[component][branch] = new_build_number build_number_file.seek(0) build_number_file.write( json.dumps(build_numbers, indent=4, sort_keys=True)) build_number_file.truncate() log.info(f'\tPush changes') push_change_commands = [ 'git add -A', f'git commit -m "Increased build number of {branch} branch for {component} to {new_build_number}"', f'git push origin HEAD:{branch}' ] for command in push_change_commands: return_code, output = cmd_exec(command, cwd=latest_version_repo_path) if return_code: log.error(output) except Exception: log.exception('Exception occurred') return False else: log.error( f'Increasing build number failed, because {latest_build_number_path} does not exist' ) return False log.info(f'Build number was increased to {new_build_number}') return True
def extract_private_infrastructure(root_dir, branch, commit_id, commit_time, manifest): log = logging.getLogger('extract_repo.extract_private_infrastructure') infrastructure_root_dir = root_dir / 'infrastructure' configs_root_dir = root_dir / 'product-configs' # We save and update repos in temporary folder and create infrastructure package from it # So, not needed extracting repo to the beginning each time original_repos_dir = root_dir / 'tmp_infrastructure' repos = MediaSdkDirectories() open_source_product_configs_repo = repos.open_source_product_configs_repo open_source_infra_repo = repos.open_source_infrastructure_repo closed_source_product_configs_repo = repos.closed_source_product_configs_repo closed_source_infra_repo = repos.closed_source_infrastructure_repo # Extract open source infrastructure and product configs extract_open_source_infrastructure(original_repos_dir, branch, commit_id, commit_time, manifest) # Extract closed source product configs extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_product_configs_repo, branch='master', commit_time=commit_time) manifest_path = original_repos_dir / closed_source_product_configs_repo / 'manifest.yml' manifest_data = Manifest(manifest_path) closed_source_infra = manifest_data.get_component('infra').get_repository( closed_source_infra_repo) # Extract closed source infrastructure extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_infra.name, branch=closed_source_infra.branch, commit_id=closed_source_infra.revision) # Event repository in infra component for private builds is product-configs, # so need to change default trigger value manifest_data.get_component('infra').build_info.set_trigger( open_source_product_configs_repo) manifest_data.save_manifest(manifest_path) log.info('-' * 50) log.info(f"Create infrastructure package") try: log.info(f"- Delete existing infrastructure") if infrastructure_root_dir.exists(): remove_directory(str(infrastructure_root_dir)) if configs_root_dir.exists(): remove_directory(str(configs_root_dir)) log.info(f"- Copy open source infrastructure") copy_tree(str(original_repos_dir / open_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Copy closed source infrastructure") copy_tree(str(original_repos_dir / closed_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Remove closed source static data") (infrastructure_root_dir / 'common' / 'static_closed_data.py').unlink() log.info(f"- Copy open source product configs") copy_tree(str(original_repos_dir / open_source_product_configs_repo), str(configs_root_dir)) log.info(f"- Copy closed source product configs") copy_tree(str(original_repos_dir / closed_source_product_configs_repo), str(configs_root_dir)) # log.info(f"Copy secrets") shutil.copyfile( str(pathlib.Path('msdk_secrets.py').absolute()), str(infrastructure_root_dir / 'common' / 'msdk_secrets.py')) except Exception: log.exception('Can not create infrastructure package') exit_script(ErrorCode.CRITICAL)
def extract_closed_source_infrastructure(root_dir, branch, commit_id, commit_time, manifest): log = logging.getLogger( 'extract_repo.extract_closed_source_infrastructure') infrastructure_root_dir = root_dir / 'infrastructure' configs_root_dir = root_dir / 'product-configs' # We save and update repos in temporary folder and create infrastructure package from it # So, not needed extracting repo to the beginning each time original_repos_dir = root_dir / 'tmp_infrastructure' repos = MediaSdkDirectories() closed_source_product_configs_repo = repos.closed_source_product_configs_repo open_source_infra_repo = repos.open_source_infrastructure_repo closed_source_infra_repo = repos.closed_source_infrastructure_repo # Extract product configs if not manifest: extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_product_configs_repo, branch=branch, commit_id=commit_id, commit_time=commit_time) manifest_data = Manifest(original_repos_dir / closed_source_product_configs_repo / 'manifest.yml') else: manifest_data = Manifest(manifest) product_conf = manifest_data.get_component('infra').get_repository( closed_source_product_configs_repo) extract_repo(root_repo_dir=original_repos_dir, repo_name=product_conf.name, branch=product_conf.branch, commit_id=product_conf.revision, commit_time=commit_time) open_source_infra = manifest_data.get_component('infra').get_repository( open_source_infra_repo) closed_source_infra = manifest_data.get_component('infra').get_repository( closed_source_infra_repo) # Extract open source infrastructure # Set proxy for access to GitHub extract_repo(root_repo_dir=original_repos_dir, repo_name=open_source_infra.name, branch=open_source_infra.branch, commit_id=open_source_infra.revision, proxy=True) # Extract closed source part of infrastructure extract_repo(root_repo_dir=original_repos_dir, repo_name=closed_source_infra.name, branch=closed_source_infra.branch, commit_id=closed_source_infra.revision) log.info('-' * 50) log.info(f"Create infrastructure package") try: log.info(f"- Delete existing infrastructure") if infrastructure_root_dir.exists(): remove_directory(str(infrastructure_root_dir)) if configs_root_dir.exists(): remove_directory(str(configs_root_dir)) log.info(f"- Copy open source infrastructure") copy_tree(str(original_repos_dir / open_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Copy closed source infrastructure") copy_tree(str(original_repos_dir / closed_source_infra_repo), str(infrastructure_root_dir)) log.info(f"- Copy product configs") copy_tree(str(original_repos_dir / closed_source_product_configs_repo), str(configs_root_dir)) # log.info(f"Copy secrets") shutil.copyfile( str(pathlib.Path('msdk_secrets.py').absolute()), str(infrastructure_root_dir / 'common' / 'msdk_secrets.py')) except Exception: log.exception('Can not create infrastructure package') exit_script(ErrorCode.CRITICAL)