def configure_remotes(self, remote_name: str, remote_url: str,
upstream_remote_name: str, upstream_remote_url: str) -> None:
"""Configure remotes names for project and upstream
:param str remote_name: Project remote name
:param str remote_url: Project remote url
:param str upstream_remote_name: Upstream remote name
:param str upstream_remote_url: Upstream remote url
"""
if not existing_git_repo(self.repo_path):
return
try:
remotes = self.repo.remotes
except GitError as err:
LOG.debug('No remotes', err)
return
else:
for remote in remotes:
if remote_url == self._remote_get_url(remote.name) and remote.name != remote_name:
self._rename_remote(remote.name, remote_name)
continue
if upstream_remote_url == self._remote_get_url(remote.name) and remote.name != upstream_remote_name:
self._rename_remote(remote.name, upstream_remote_name)
self._compare_remotes(remote_name, remote_url, upstream_remote_name, upstream_remote_url)
def new_commits_count(self, upstream: bool = False) -> int:
"""Returns the number of new commits
:param bool upstream: Whether to find number of new upstream or local commits
:return: Int number of new commits
"""
try:
local_branch = self.repo.active_branch
except (GitError, TypeError) as err:
LOG.debug(error=err)
return 0
else:
tracking_branch = local_branch.tracking_branch()
if local_branch is None or tracking_branch is None:
return 0
try:
commits = f'{local_branch.commit.hexsha}...{tracking_branch.commit.hexsha}'
rev_list_count = self.repo.git.rev_list(
'--count', '--left-right', commits)
except (GitError, ValueError) as err:
LOG.debug(error=err)
return 0
else:
index = 1 if upstream else 0
return int(str(rev_list_count).split()[index])
async def run_sync(func: Callable, limit: trio.CapacityLimiter,
project: ResolvedProject, progress: tqdm) -> None:
LOG.debug(f'START PARALLEL {project.name}')
await trio.to_thread.run_sync(func, limiter=limit)
limit.release_on_behalf_of(project)
progress.update()
LOG.debug(f'END PARALLEL {project.name}')
def herd_tag(self, url: str, tag: str, depth: int = 0,
rebase: bool = False, config: Optional[GitConfig] = None) -> None:
"""Herd tag
:param str url: URL of repo
:param str tag: Tag name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool rebase: Whether to use rebase instead of pulling latest changes
:param Optional[GitConfig] config: Custom git config
"""
fetch = depth != 0
if not existing_git_repo(self.repo_path):
self._init_repo()
self._create_remote(self.remote, url, remove_dir=True)
try:
self._checkout_new_repo_tag(tag, self.remote, depth)
except Exception as err:
LOG.debug('Failed checkout new repo tag', err)
self.herd(url, depth=depth, fetch=fetch, rebase=rebase)
return
self.install_project_git_herd_alias()
if config is not None:
self._update_git_config(config)
try:
self.fetch(self.remote, ref=GitRef(tag=tag), depth=depth)
self._checkout_tag(tag)
except Exception as err:
LOG.debug('Failed fetch and checkout tag', err)
self.herd(url, depth=depth, fetch=fetch, rebase=rebase)
def start(self, remote: str, branch: str, depth: int, tracking: bool) -> None:
"""Start new branch in repository and checkout
:param str remote: Remote name
:param str branch: Local branch name to create
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool tracking: Whether to create a remote branch with tracking relationship
"""
if branch not in self.repo.heads:
if not is_offline():
self.fetch(remote, ref=GitRef(branch=branch), depth=depth)
try:
self._create_branch_local(branch)
self._checkout_branch_local(branch)
except BaseException as err:
LOG.debug('Failed to create and checkout branch', err)
raise
else:
CONSOLE.stdout(f' - {fmt.ref(branch)} already exists')
if self._is_branch_checked_out(branch):
CONSOLE.stdout(' - On correct branch')
else:
self._checkout_branch_local(branch)
if tracking and not is_offline():
self._create_branch_remote_tracking(branch, remote, depth)
def _get_remote_tag(self,
tag: str,
remote: str,
depth: int = 0,
remove_dir: bool = False) -> Optional[Tag]:
"""Returns Tag object
:param str tag: Tag name
:param str remote: Remote name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool remove_dir: Whether to remove the directory if commands fail
:return: GitPython Tag object if it exists, otherwise None
"""
self._remote(remote, remove_dir=remove_dir)
self.fetch(remote,
depth=depth,
ref=GitRef(tag=tag),
remove_dir=remove_dir)
try:
return self.repo.tags[tag]
except (GitError, IndexError) as err:
LOG.error(f'No existing tag {fmt.ref(tag)}')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
LOG.debug(error=err)
return None
except BaseException:
LOG.error('Failed to get tag')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
def is_lfs_installed(self) -> bool:
"""Check whether git lfs hooks are installed"""
try:
self.repo.git.config('--get', 'filter.lfs.smudge')
self.repo.git.config('--get', 'filter.lfs.clean')
self.repo.git.config('--get', 'filter.lfs.process')
self.repo.git.config('--get', 'filter.lfs.required')
except GitError as err:
LOG.debug(error=err)
return False
else:
return True
def get_default_branch_from_remote(url: str) -> Optional[str]:
"""Get default branch from remote repo"""
try:
command = ['git', 'ls-remote', '--symref', url, 'HEAD']
result = execute_command(command, Path.cwd(), print_output=False)
output: str = result.stdout
output_list = output.split()
branch = [remove_prefix(chunk, 'refs/heads/') for chunk in output_list if chunk.startswith('refs/heads/')]
return branch[0]
except CalledProcessError as err:
LOG.debug('Failed to get default branch from remote git repo', err)
return None
def has_remote_branch(self, branch: str, remote: str) -> bool:
"""Check if remote branch exists
:param str branch: Branch name
:param str remote: Remote name
:return: True, if remote branch exists
"""
try:
origin = self.repo.remotes[remote]
return branch in origin.refs
except (GitError, IndexError) as err:
LOG.debug(error=err)
return False
def get_default_branch_from_local(repo_path: Path, remote: str) -> Optional[str]:
"""Get default branch from local repo"""
try:
command = ['git', 'symbolic-ref', f'refs/remotes/{remote}/HEAD']
result = execute_command(command, repo_path, print_output=False)
output: str = result.stdout
output_list = output.split()
branch = [remove_prefix(chunk, f'refs/remotes/{remote}/') for chunk in output_list
if chunk.startswith(f'refs/remotes/{remote}/')]
return branch[0]
except CalledProcessError as err:
LOG.debug('Failed to get default branch from local git repo', err)
return None
def _is_branch_checked_out(self, branch: str) -> bool:
"""Check if branch is checked out
:param str branch: Branch name
:return: True, if branch is checked out
"""
try:
default_branch = self.repo.heads[branch]
is_detached = self.repo.head.is_detached
same_branch = self.repo.head.ref == default_branch
return not is_detached and same_branch
except (GitError, TypeError) as err:
LOG.debug(error=err)
return False
def _run_forall_command(self, command: str, env: dict,
ignore_errors: bool) -> None:
"""Run command or script in project directory
:param str command: Command to run
:param dict env: Environment variables
:param bool ignore_errors: Whether to exit if command returns a non-zero exit code
"""
CONSOLE.stdout(fmt.command(command))
try:
execute_forall_command(command, self.full_path, env)
except CalledProcessError as err:
if ignore_errors:
LOG.debug(f'Command failed: {command}', err)
else:
LOG.error(f'Command failed: {command}')
raise
def herd_remote(self, url: str, remote: str, branch: Optional[str] = None) -> None:
"""Herd remote repo
:param str url: URL of repo
:param str remote: Remote name
:param Optional[str] branch: Branch name
"""
self._create_remote(remote, url)
if branch is None:
self.fetch(remote, ref=self.default_ref)
return
try:
self.fetch(remote, ref=GitRef(branch=branch))
except Exception as err:
LOG.debug('Failed fetch', err)
self.fetch(remote, ref=self.default_ref)
def fetch(self,
remote: str,
ref: Optional[GitRef] = None,
depth: int = 0,
remove_dir: bool = False,
allow_failure: bool = False) -> None:
"""Fetch from a specific remote ref
:param str remote: Remote name
:param Optional[GitRef] ref: Ref to fetch
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool remove_dir: Whether to remove the directory if commands fail
:param bool allow_failure: Whether to allow failure
"""
if depth == 0 or ref is None:
CONSOLE.stdout(f' - Fetch from {fmt.remote(remote)}')
error_message = f'Failed to fetch from remote {fmt.remote(remote)}'
else:
CONSOLE.stdout(
f' - Fetch from {fmt.remote(remote)} {fmt.ref(ref.short_ref)}')
error_message = f'Failed to fetch from {fmt.remote(remote)} {fmt.ref(ref.short_ref)}'
try:
if depth == 0:
execute_command(f'git fetch {remote} --prune --tags',
self.repo_path)
elif ref is None:
command = f'git fetch {remote} --depth {depth} --prune --tags'
execute_command(command, self.repo_path)
else:
command = f'git fetch {remote} {ref.short_ref} --depth {depth} --prune --tags'
execute_command(command, self.repo_path)
except BaseException as err:
LOG.error(error_message)
if remove_dir:
remove_directory(self.repo_path, check=False)
if allow_failure:
LOG.debug(error=err)
return
raise
def __init__(self):
"""ClowderController __init__
:raise MissingYamlError:
"""
self.error: Optional[Exception] = None
self._initialize_properties()
try:
if ENVIRONMENT.clowder_yaml is None:
raise MissingYamlError(
f"{Path('clowder.yml')} appears to be missing")
yaml = load_yaml_file(ENVIRONMENT.clowder_yaml,
ENVIRONMENT.clowder_dir)
validate_yaml_file(yaml, ENVIRONMENT.clowder_yaml)
self._clowder = ClowderBase(yaml)
# Register all sources as we come across them
defaults = self._clowder.defaults
if defaults is not None and defaults.source is not None:
SOURCE_CONTROLLER.add_source(defaults.source)
sources = self._clowder.sources
if sources is not None:
for s in sources:
SOURCE_CONTROLLER.add_source(s)
projects = self._clowder.clowder.projects
sections = self._clowder.clowder.sections
if projects is None:
projects = [p for s in sections for p in s.projects]
for project in projects:
SOURCE_CONTROLLER.add_source(project.source)
if project.upstream is not None:
SOURCE_CONTROLLER.add_source(project.upstream.source)
# Validate all source names have a defined source with url
SOURCE_CONTROLLER.validate_sources()
if sections is None:
resolved_projects = [
ResolvedProject(p,
defaults=defaults,
protocol=self._clowder.protocol)
for p in projects
]
else:
resolved_projects = [
ResolvedProject(p,
defaults=defaults,
section=s,
protocol=self._clowder.protocol)
for s in sections for p in s.projects
]
self.projects = tuple(
sorted(resolved_projects, key=lambda p: p.name))
self._update_properties()
self._populate_default_branches()
except Exception as err:
LOG.debug('Failed to init clowder controller')
self.error = err
self._initialize_properties()