def create_clowder_repo(self, url: str, branch: str, depth: int = 0) -> None:
"""Clone clowder git repo from url at path
:param str url: URL of repo
:param str branch: Branch name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:raise ExistingFileError:
"""
if existing_git_repo(self.repo_path):
# TODO: Throw error if repo doesn't match one trying to create
return
if self.repo_path.is_dir():
try:
self.repo_path.rmdir()
except OSError:
LOG.error(f"Directory already exists at {fmt.path(self.repo_path)}")
raise
if self.repo_path.is_symlink():
remove_file(self.repo_path)
else:
from clowder.environment import ENVIRONMENT
if ENVIRONMENT.existing_clowder_repo_file_error:
raise ENVIRONMENT.existing_clowder_repo_file_error
self._init_repo()
self._create_remote(self.remote, url, remove_dir=True)
self._checkout_new_repo_branch(branch, depth)
def symlink_clowder_yaml(source: Path, target: Path) -> None:
"""Force symlink creation
:param Path source: File to create symlink pointing to
:param Path target: Symlink location
:raise ExistingFileError:
:raise MissingSourceError:
"""
if not target.is_symlink() and target.is_file():
raise ExistingFileError(
f"Found non-symlink file {fmt.path(target)} at target path")
if not Path(target.parent / source).exists():
raise MissingSourceError(
f"Symlink source {fmt.path(source)} appears to be missing")
if target.is_symlink():
remove_file(target)
try:
path = target.parent
fd = os.open(path, os.O_DIRECTORY)
os.symlink(source, target, dir_fd=fd)
os.close(fd)
except OSError:
LOG.error(
f"Failed to symlink file {fmt.path(target)} -> {fmt.path(source)}")
raise
def configure_remotes(self, remote_name: str, remote_url: str,
upstream_remote_name: str, upstream_remote_url: str) -> None:
"""Configure remotes names for project and upstream
:param str remote_name: Project remote name
:param str remote_url: Project remote url
:param str upstream_remote_name: Upstream remote name
:param str upstream_remote_url: Upstream remote url
"""
if not existing_git_repo(self.repo_path):
return
try:
remotes = self.repo.remotes
except GitError as err:
LOG.debug('No remotes', err)
return
else:
for remote in remotes:
if remote_url == self._remote_get_url(remote.name) and remote.name != remote_name:
self._rename_remote(remote.name, remote_name)
continue
if upstream_remote_url == self._remote_get_url(remote.name) and remote.name != upstream_remote_name:
self._rename_remote(remote.name, upstream_remote_name)
self._compare_remotes(remote_name, remote_url, upstream_remote_name, upstream_remote_url)
def prune_branch_local(self, branch: str, force: bool) -> None:
"""Prune local branch
:param str branch: Branch name to delete
:param bool force: Force delete branch
"""
if branch not in self.repo.heads:
CONSOLE.stdout(f" - Local branch {fmt.ref(branch)} doesn't exist")
return
prune_branch = self.repo.heads[branch]
if self.repo.head.ref == prune_branch:
try:
CONSOLE.stdout(f' - Checkout ref {fmt.ref(self.default_ref.short_ref)}')
self.repo.git.checkout(self.default_ref.short_ref)
except GitError:
LOG.error(f'Failed to checkout ref {fmt.ref(self.default_ref.short_ref)}')
raise
try:
CONSOLE.stdout(f' - Delete local branch {fmt.ref(branch)}')
self.repo.delete_head(branch, force=force)
except GitError:
LOG.error(f'Failed to delete local branch {fmt.ref(branch)}')
raise
def start(self, remote: str, branch: str, depth: int, tracking: bool) -> None:
"""Start new branch in repository and checkout
:param str remote: Remote name
:param str branch: Local branch name to create
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool tracking: Whether to create a remote branch with tracking relationship
"""
if branch not in self.repo.heads:
if not is_offline():
self.fetch(remote, ref=GitRef(branch=branch), depth=depth)
try:
self._create_branch_local(branch)
self._checkout_branch_local(branch)
except BaseException as err:
LOG.debug('Failed to create and checkout branch', err)
raise
else:
CONSOLE.stdout(f' - {fmt.ref(branch)} already exists')
if self._is_branch_checked_out(branch):
CONSOLE.stdout(' - On correct branch')
else:
self._checkout_branch_local(branch)
if tracking and not is_offline():
self._create_branch_remote_tracking(branch, remote, depth)
def _checkout_new_repo_tag(self,
tag: str,
remote: str,
depth: int,
remove_dir: bool = False) -> None:
"""Checkout tag or fail and delete repo if it doesn't exist
:param str tag: Tag name
:param str remote: Remote name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool remove_dir: Whether to remove the directory if commands fail
"""
remote_tag = self._get_remote_tag(tag,
remote,
depth=depth,
remove_dir=remove_dir)
if remote_tag is None:
return
try:
CONSOLE.stdout(f' - Checkout tag {fmt.ref(tag)}')
self.repo.git.checkout(remote_tag)
except BaseException:
LOG.error(f'Failed to checkout tag {fmt.ref(tag)}')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
def herd_tag(self, url: str, tag: str, depth: int = 0,
rebase: bool = False, config: Optional[GitConfig] = None) -> None:
"""Herd tag
:param str url: URL of repo
:param str tag: Tag name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool rebase: Whether to use rebase instead of pulling latest changes
:param Optional[GitConfig] config: Custom git config
"""
fetch = depth != 0
if not existing_git_repo(self.repo_path):
self._init_repo()
self._create_remote(self.remote, url, remove_dir=True)
try:
self._checkout_new_repo_tag(tag, self.remote, depth)
except Exception as err:
LOG.debug('Failed checkout new repo tag', err)
self.herd(url, depth=depth, fetch=fetch, rebase=rebase)
return
self.install_project_git_herd_alias()
if config is not None:
self._update_git_config(config)
try:
self.fetch(self.remote, ref=GitRef(tag=tag), depth=depth)
self._checkout_tag(tag)
except Exception as err:
LOG.debug('Failed fetch and checkout tag', err)
self.herd(url, depth=depth, fetch=fetch, rebase=rebase)
def _create_branch_local_tracking(self,
branch: str,
remote: str,
depth: int,
fetch: bool = True,
remove_dir: bool = False) -> None:
"""Create and checkout tracking branch
:param str branch: Branch name
:param str remote: Remote name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool fetch: Whether to fetch before creating branch
:param bool remove_dir: Whether to remove the directory if commands fail
"""
origin = self._remote(remote, remove_dir=remove_dir)
if fetch:
self.fetch(remote,
depth=depth,
ref=GitRef(branch=branch),
remove_dir=remove_dir)
try:
CONSOLE.stdout(f' - Create branch {fmt.ref(branch)}')
self.repo.create_head(branch, origin.refs[branch])
except BaseException:
LOG.error(f'Failed to create branch {fmt.ref(branch)}')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
else:
self._set_tracking_branch(remote, branch, remove_dir=remove_dir)
self._checkout_branch_local(branch, remove_dir=remove_dir)
def new_commits_count(self, upstream: bool = False) -> int:
"""Returns the number of new commits
:param bool upstream: Whether to find number of new upstream or local commits
:return: Int number of new commits
"""
try:
local_branch = self.repo.active_branch
except (GitError, TypeError) as err:
LOG.debug(error=err)
return 0
else:
tracking_branch = local_branch.tracking_branch()
if local_branch is None or tracking_branch is None:
return 0
try:
commits = f'{local_branch.commit.hexsha}...{tracking_branch.commit.hexsha}'
rev_list_count = self.repo.git.rev_list(
'--count', '--left-right', commits)
except (GitError, ValueError) as err:
LOG.debug(error=err)
return 0
else:
index = 1 if upstream else 0
return int(str(rev_list_count).split()[index])
async def run_sync(func: Callable, limit: trio.CapacityLimiter,
project: ResolvedProject, progress: tqdm) -> None:
LOG.debug(f'START PARALLEL {project.name}')
await trio.to_thread.run_sync(func, limiter=limit)
limit.release_on_behalf_of(project)
progress.update()
LOG.debug(f'END PARALLEL {project.name}')
def main() -> None:
"""Clowder command CLI main function"""
try:
parser = create_parsers()
argcomplete.autocomplete(parser)
args = parser.parse_args()
if 'projects' in args:
if isinstance(args.projects, str):
args.projects = [args.projects]
if args.debug:
LOG.level = LOG.DEBUG
args.func(args)
except CalledProcessError as err:
LOG.error(error=err)
exit(err.returncode)
except OSError as err:
LOG.error(error=err)
exit(err.errno)
except SystemExit as err:
if err.code == 0:
exit()
LOG.error(error=err)
exit(err.code)
except KeyboardInterrupt:
LOG.error('** KeyboardInterrupt **')
exit(1)
except BaseException as err:
LOG.error(error=err)
exit(1)
def current_timestamp(self) -> str:
"""Current timestamp of HEAD commit"""
try:
return self.repo.git.log('-1', '--format=%cI')
except GitError:
LOG.error('Failed to find current timestamp')
raise
def pull_lfs(self) -> None:
"""Pull lfs files"""
try:
CONSOLE.stdout(' - Pull git lfs files')
self.repo.git.lfs('pull')
except GitError:
LOG.error('Failed to pull git lfs files')
raise
def pull(self) -> None:
"""Pull upstream changes"""
try:
CONSOLE.stdout(' - Pull latest changes')
CONSOLE.stdout(self.repo.git.pull())
except GitError:
LOG.error('Failed to pull latest changes')
raise
def install_lfs_hooks(self) -> None:
"""Install git lfs hooks"""
CONSOLE.stdout(" - Update git lfs hooks")
try:
self.repo.git.lfs('install', '--local')
except GitError:
LOG.error('Failed to update git lfs hooks')
raise
def push(self) -> None:
"""Push changes"""
try:
CONSOLE.stdout(' - Push local changes')
CONSOLE.stdout(self.repo.git.push())
except GitError:
LOG.error('Failed to push local changes')
raise
def _abort_rebase(self) -> None:
"""Abort rebase"""
if not self._is_rebase_in_progress:
return
try:
self.repo.git.rebase('--abort')
except GitError:
LOG.error('Failed to abort rebase')
raise
def _clean(self, args: str) -> None:
"""Clean git directory
:param str args: Git clean args
"""
try:
self.repo.git.clean(args)
except GitError:
LOG.error('Failed to clean git repo')
raise
def commit(self, message: str) -> None:
"""Commit current changes
:param str message: Git commit message
"""
try:
CONSOLE.stdout(' - Commit current changes')
CONSOLE.stdout(self.repo.git.commit(message=message))
except GitError:
LOG.error('Failed to commit current changes')
raise
def _create_branch_local(self, branch: str) -> None:
"""Create local branch
:param str branch: Branch name
"""
try:
CONSOLE.stdout(f' - Create branch {fmt.ref(branch)}')
self.repo.create_head(branch)
except GitError:
LOG.error(f'Failed to create branch {fmt.ref(branch)}')
raise
def is_lfs_installed(self) -> bool:
"""Check whether git lfs hooks are installed"""
try:
self.repo.git.config('--get', 'filter.lfs.smudge')
self.repo.git.config('--get', 'filter.lfs.clean')
self.repo.git.config('--get', 'filter.lfs.process')
self.repo.git.config('--get', 'filter.lfs.required')
except GitError as err:
LOG.debug(error=err)
return False
else:
return True
def _print_has_remote_branch_message(self, branch: str) -> None:
"""Print output message for existing remote branch
:param str branch: Branch name
"""
try:
self.repo.git.config('--get', 'branch.' + branch + '.merge')
CONSOLE.stdout(
f' - Tracking branch {fmt.ref(branch)} already exists')
except GitError:
LOG.error(f'Remote branch {fmt.ref(branch)} already exists')
raise
def remove_directory(dir_path: Path, check: bool = True) -> None:
"""Remove directory at path
:param str dir_path: Path to directory to remove
:param bool check: Whether to raise errors
"""
try:
shutil.rmtree(dir_path)
except shutil.Error:
LOG.error(f"Failed to remove directory {fmt.path(dir_path)}")
if check:
raise
def _print_yaml(yaml_file: Path) -> None:
"""Private print current clowder yaml file
:param Path yaml_file: Path to yaml file
"""
try:
with yaml_file.open() as raw_file:
contents = raw_file.read()
CONSOLE.stdout(contents.rstrip())
except IOError:
LOG.error(f"Failed to open file '{yaml_file}'")
raise
def get_default_branch_from_remote(url: str) -> Optional[str]:
"""Get default branch from remote repo"""
try:
command = ['git', 'ls-remote', '--symref', url, 'HEAD']
result = execute_command(command, Path.cwd(), print_output=False)
output: str = result.stdout
output_list = output.split()
branch = [remove_prefix(chunk, 'refs/heads/') for chunk in output_list if chunk.startswith('refs/heads/')]
return branch[0]
except CalledProcessError as err:
LOG.debug('Failed to get default branch from remote git repo', err)
return None
def _create_repo(self) -> Repo:
"""Create Repo instance for self.repo_path
:return: GitPython Repo instance
"""
try:
return Repo(self.repo_path)
except GitError:
LOG.error(
f"Failed to create Repo instance for {fmt.path(self.repo_path)}"
)
raise
def status_verbose(self) -> None:
"""Print git status
Equivalent to: ``git status -vv``
"""
command = 'git status -vv'
CONSOLE.stdout(fmt.command(command))
try:
execute_command(command, self.repo_path)
except CalledProcessError:
LOG.error('Failed to print verbose status')
raise
def _pull(self, remote: str, branch: str) -> None:
"""Pull from remote branch
:param str remote: Remote name
:param str branch: Branch name
"""
CONSOLE.stdout(f' - Pull from {fmt.remote(remote)} {fmt.ref(branch)}')
try:
execute_command(f"git pull {remote} {branch}", self.repo_path)
except CalledProcessError:
LOG.error(
f'Failed to pull from {fmt.remote(remote)} {fmt.ref(branch)}')
raise
def has_remote_branch(self, branch: str, remote: str) -> bool:
"""Check if remote branch exists
:param str branch: Branch name
:param str remote: Remote name
:return: True, if remote branch exists
"""
try:
origin = self.repo.remotes[remote]
return branch in origin.refs
except (GitError, IndexError) as err:
LOG.debug(error=err)
return False
def git_config_add_local(self, variable: str, value: str) -> None:
"""Add local git config value for given variable key
:param str variable: Fully qualified git config variable
:param str value: Git config value
"""
try:
self.repo.git.config('--local', '--add', variable, value)
except GitError:
LOG.error(
f'Failed to add local git config value {value} for variable {variable}'
)
raise