def _clean(self):
""" Clean temporary directory """
self._log.info(f'Cleaning {self._tmp_dir}')
try:
if self._tmp_dir.exists():
remove_directory(str(self._tmp_dir))
self._log.info(f"Removed old repository in {self._tmp_dir}")
self._tmp_dir.mkdir(exist_ok=True)
except Exception as e:
self._log.exception(f'Failed to clean {self._tmp_dir}: %s', e)
return False
return True
def clone(self):
"""
Clone repo
:return: None
"""
# checking correctness of git repository
# if dir is not repository, it will be removed
if self.local_repo_dir.exists():
try:
git.Repo(str(self.local_repo_dir))
except git.InvalidGitRepositoryError:
self.log.info('Remove broken repo %s', self.local_repo_dir)
remove_directory(self.local_repo_dir)
if not self.local_repo_dir.exists():
self.log.info("Clone repo " + self.repo_name)
git.Git().clone(self.url, str(self.local_repo_dir))
def extract_private_infrastructure(root_dir, branch, commit_id, commit_time):
log = logging.getLogger('extract_repo.extract_private_infrastructure')
infrastructure_root_dir = root_dir / 'infrastructure'
# We save and update repos in temporary folder and create infrastructure package from it
# So, not needed extracting repo to the beginning each time
original_repos_dir = root_dir / 'tmp_infrastructure'
repos = MediaSdkDirectories()
open_source_product_configs_repo = repos.open_source_product_configs_repo
open_source_infra_repo = repos.open_source_infrastructure_repo
closed_source_product_configs_repo = repos.closed_source_product_configs_repo
closed_source_infra_repo = repos.closed_source_infrastructure_repo
# Extract open source infrastructure and product configs
extract_open_source_infrastructure(original_repos_dir, branch, commit_id,
commit_time)
# Extract closed source product configs
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_product_configs_repo,
branch='master',
commit_time=commit_time)
# Get revision of closed source infrastructure from closed source product configs repo
configs_dir = original_repos_dir / closed_source_product_configs_repo
sys.path.append(str(configs_dir))
import infrastructure_version
infrastructure_version = reload(infrastructure_version)
closed_source_infra_version = infrastructure_version.CLOSED_SOURCE
# Extract closed source infrastructure
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_infra_repo,
branch=closed_source_infra_version['branch'],
commit_id=closed_source_infra_version['commit_id'])
log.info('-' * 50)
log.info(f"Create infrastructure package")
try:
log.info(f"- Delete existing infrastructure")
if infrastructure_root_dir.exists():
remove_directory(str(infrastructure_root_dir))
log.info(f"- Copy open source infrastructure")
copy_tree(str(original_repos_dir / open_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Copy closed source infrastructure")
copy_tree(str(original_repos_dir / closed_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Remove closed source static data")
(infrastructure_root_dir / 'common' / 'static_closed_data.py').unlink()
log.info(f"- Copy open source product configs")
copy_tree(
str(original_repos_dir / open_source_product_configs_repo),
str(infrastructure_root_dir / open_source_product_configs_repo))
log.info(f"- Copy closed source product configs")
copy_tree(
str(original_repos_dir / closed_source_product_configs_repo),
str(infrastructure_root_dir / open_source_product_configs_repo))
# log.info(f"Copy secrets")
shutil.copyfile(
str(pathlib.Path('msdk_secrets.py').absolute()),
str(infrastructure_root_dir / 'common' / 'msdk_secrets.py'))
except Exception:
log.exception('Can not create infrastructure package')
exit_script(ErrorCode.CRITICAL)
def increase_build_number(local_repo_path, component, branch):
"""
Increase build number by 1 in remote repository, if it is the same for local and remote repositories
This condition is needed to avoid increasing build number while rebuilds
Function extracts product-configs repo in following layout to push change to remote repository
../tmp/product-configs
../origin_repo_path
:param local_repo_path: path to local repository with "build_numbers.json" file
:type local_repo_path: String | pathlib.Path
:param component: Component name. Need for finding certain build number
:type component: String
:param branch: Name of branch. Need for finding certain build number
:type branch: String
"""
log = logging.getLogger('build_number.increase_build_number')
log.info(f'Increasing build number in {branch} branch for {component}')
build_numbers_file = 'build_numbers.json'
log.info(f'Get build number from local repository')
current_build_number = get_build_number(
repo_path=pathlib.Path(local_repo_path),
component=component,
branch=branch)
if current_build_number == 0:
log.error(
f'Local build number must not be 0\n'
f'Check that {pathlib.Path(local_repo_path) / build_numbers_file} contains appropriate {branch} branch for {component}'
)
return False
repo_name = pathlib.Path(local_repo_path).name
temp_dir = (pathlib.Path(local_repo_path) / '..' / 'tmp').resolve()
latest_version_repo_path = temp_dir / repo_name
latest_build_number_path = latest_version_repo_path / build_numbers_file
if temp_dir.exists():
log.info(f"Remove old repository in {temp_dir}")
remove_directory(str(temp_dir))
temp_dir.mkdir(exist_ok=True)
log.warning(f'Redefine {branch} to "master"')
# TODO: use extract_repo from git_worker in one_ci_dev branch
branch = 'master'
extract_repo(root_repo_dir=temp_dir,
repo_name=repo_name,
branch=branch,
commit_id='HEAD')
log.info(
f'Getting build number from HEAD of {branch} branch for repo in {latest_version_repo_path}'
)
latest_git_build_number = get_build_number(
repo_path=latest_version_repo_path, component=component, branch=branch)
if current_build_number != latest_git_build_number:
log.warning(
f'Build numbers in remote ({latest_git_build_number}) and local ({current_build_number}) repositories are not equal\n'
f'It maybe because this is rebuild of old build for which build number already has been increased\n'
f'Stop operation')
return False
log.info('Increasing build number')
if latest_build_number_path.exists():
try:
log.info(f'\tChanging build numbers file')
with latest_build_number_path.open('r+') as build_number_file:
build_numbers = json.load(build_number_file)
new_build_number = build_numbers[component][branch] + 1
build_numbers[component][branch] = new_build_number
build_number_file.seek(0)
build_number_file.write(
json.dumps(build_numbers, indent=4, sort_keys=True))
build_number_file.truncate()
log.info(f'\tPush changes')
push_change_commands = [
'git add -A',
f'git commit -m "Increased build number of {branch} branch for {component} to {new_build_number}"',
f'git push origin HEAD:{branch}'
]
for command in push_change_commands:
return_code, output = cmd_exec(command,
cwd=latest_version_repo_path)
if return_code:
log.error(output)
except Exception:
log.exception('Exception occurred')
return False
else:
log.error(
f'Increasing build number failed, because {latest_build_number_path} does not exist'
)
return False
log.info(f'Build number was increased to {new_build_number}')
return True
def extract_private_infrastructure(root_dir, branch, commit_id, commit_time,
manifest):
log = logging.getLogger('extract_repo.extract_private_infrastructure')
infrastructure_root_dir = root_dir / 'infrastructure'
configs_root_dir = root_dir / 'product-configs'
# We save and update repos in temporary folder and create infrastructure package from it
# So, not needed extracting repo to the beginning each time
original_repos_dir = root_dir / 'tmp_infrastructure'
repos = MediaSdkDirectories()
open_source_product_configs_repo = repos.open_source_product_configs_repo
open_source_infra_repo = repos.open_source_infrastructure_repo
closed_source_product_configs_repo = repos.closed_source_product_configs_repo
closed_source_infra_repo = repos.closed_source_infrastructure_repo
# Extract open source infrastructure and product configs
extract_open_source_infrastructure(original_repos_dir, branch, commit_id,
commit_time, manifest)
# Extract closed source product configs
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_product_configs_repo,
branch='master',
commit_time=commit_time)
manifest_path = original_repos_dir / closed_source_product_configs_repo / 'manifest.yml'
manifest_data = Manifest(manifest_path)
closed_source_infra = manifest_data.get_component('infra').get_repository(
closed_source_infra_repo)
# Extract closed source infrastructure
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_infra.name,
branch=closed_source_infra.branch,
commit_id=closed_source_infra.revision)
# Event repository in infra component for private builds is product-configs,
# so need to change default trigger value
manifest_data.get_component('infra').build_info.set_trigger(
open_source_product_configs_repo)
manifest_data.save_manifest(manifest_path)
log.info('-' * 50)
log.info(f"Create infrastructure package")
try:
log.info(f"- Delete existing infrastructure")
if infrastructure_root_dir.exists():
remove_directory(str(infrastructure_root_dir))
if configs_root_dir.exists():
remove_directory(str(configs_root_dir))
log.info(f"- Copy open source infrastructure")
copy_tree(str(original_repos_dir / open_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Copy closed source infrastructure")
copy_tree(str(original_repos_dir / closed_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Remove closed source static data")
(infrastructure_root_dir / 'common' / 'static_closed_data.py').unlink()
log.info(f"- Copy open source product configs")
copy_tree(str(original_repos_dir / open_source_product_configs_repo),
str(configs_root_dir))
log.info(f"- Copy closed source product configs")
copy_tree(str(original_repos_dir / closed_source_product_configs_repo),
str(configs_root_dir))
# log.info(f"Copy secrets")
shutil.copyfile(
str(pathlib.Path('msdk_secrets.py').absolute()),
str(infrastructure_root_dir / 'common' / 'msdk_secrets.py'))
except Exception:
log.exception('Can not create infrastructure package')
exit_script(ErrorCode.CRITICAL)
def extract_closed_source_infrastructure(root_dir, branch, commit_id,
commit_time, manifest):
log = logging.getLogger(
'extract_repo.extract_closed_source_infrastructure')
infrastructure_root_dir = root_dir / 'infrastructure'
configs_root_dir = root_dir / 'product-configs'
# We save and update repos in temporary folder and create infrastructure package from it
# So, not needed extracting repo to the beginning each time
original_repos_dir = root_dir / 'tmp_infrastructure'
repos = MediaSdkDirectories()
closed_source_product_configs_repo = repos.closed_source_product_configs_repo
open_source_infra_repo = repos.open_source_infrastructure_repo
closed_source_infra_repo = repos.closed_source_infrastructure_repo
# Extract product configs
if not manifest:
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_product_configs_repo,
branch=branch,
commit_id=commit_id,
commit_time=commit_time)
manifest_data = Manifest(original_repos_dir /
closed_source_product_configs_repo /
'manifest.yml')
else:
manifest_data = Manifest(manifest)
product_conf = manifest_data.get_component('infra').get_repository(
closed_source_product_configs_repo)
extract_repo(root_repo_dir=original_repos_dir,
repo_name=product_conf.name,
branch=product_conf.branch,
commit_id=product_conf.revision,
commit_time=commit_time)
open_source_infra = manifest_data.get_component('infra').get_repository(
open_source_infra_repo)
closed_source_infra = manifest_data.get_component('infra').get_repository(
closed_source_infra_repo)
# Extract open source infrastructure
# Set proxy for access to GitHub
extract_repo(root_repo_dir=original_repos_dir,
repo_name=open_source_infra.name,
branch=open_source_infra.branch,
commit_id=open_source_infra.revision,
proxy=True)
# Extract closed source part of infrastructure
extract_repo(root_repo_dir=original_repos_dir,
repo_name=closed_source_infra.name,
branch=closed_source_infra.branch,
commit_id=closed_source_infra.revision)
log.info('-' * 50)
log.info(f"Create infrastructure package")
try:
log.info(f"- Delete existing infrastructure")
if infrastructure_root_dir.exists():
remove_directory(str(infrastructure_root_dir))
if configs_root_dir.exists():
remove_directory(str(configs_root_dir))
log.info(f"- Copy open source infrastructure")
copy_tree(str(original_repos_dir / open_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Copy closed source infrastructure")
copy_tree(str(original_repos_dir / closed_source_infra_repo),
str(infrastructure_root_dir))
log.info(f"- Copy product configs")
copy_tree(str(original_repos_dir / closed_source_product_configs_repo),
str(configs_root_dir))
# log.info(f"Copy secrets")
shutil.copyfile(
str(pathlib.Path('msdk_secrets.py').absolute()),
str(infrastructure_root_dir / 'common' / 'msdk_secrets.py'))
except Exception:
log.exception('Can not create infrastructure package')
exit_script(ErrorCode.CRITICAL)
