def symlink_clowder_yaml(source: Path, target: Path) -> None:
"""Force symlink creation
:param Path source: File to create symlink pointing to
:param Path target: Symlink location
:raise ExistingFileError:
:raise MissingSourceError:
"""
if not target.is_symlink() and target.is_file():
raise ExistingFileError(
f"Found non-symlink file {fmt.path(target)} at target path")
if not Path(target.parent / source).exists():
raise MissingSourceError(
f"Symlink source {fmt.path(source)} appears to be missing")
if target.is_symlink():
remove_file(target)
try:
path = target.parent
fd = os.open(path, os.O_DIRECTORY)
os.symlink(source, target, dir_fd=fd)
os.close(fd)
except OSError:
LOG.error(
f"Failed to symlink file {fmt.path(target)} -> {fmt.path(source)}")
raise
def create_clowder_repo(self, url: str, branch: str, depth: int = 0) -> None:
"""Clone clowder git repo from url at path
:param str url: URL of repo
:param str branch: Branch name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:raise ExistingFileError:
"""
if existing_git_repo(self.repo_path):
# TODO: Throw error if repo doesn't match one trying to create
return
if self.repo_path.is_dir():
try:
self.repo_path.rmdir()
except OSError:
LOG.error(f"Directory already exists at {fmt.path(self.repo_path)}")
raise
if self.repo_path.is_symlink():
remove_file(self.repo_path)
else:
from clowder.environment import ENVIRONMENT
if ENVIRONMENT.existing_clowder_repo_file_error:
raise ENVIRONMENT.existing_clowder_repo_file_error
self._init_repo()
self._create_remote(self.remote, url, remove_dir=True)
self._checkout_new_repo_branch(branch, depth)
def main() -> None:
"""Clowder command CLI main function"""
try:
parser = create_parsers()
argcomplete.autocomplete(parser)
args = parser.parse_args()
if 'projects' in args:
if isinstance(args.projects, str):
args.projects = [args.projects]
if args.debug:
LOG.level = LOG.DEBUG
args.func(args)
except CalledProcessError as err:
LOG.error(error=err)
exit(err.returncode)
except OSError as err:
LOG.error(error=err)
exit(err.errno)
except SystemExit as err:
if err.code == 0:
exit()
LOG.error(error=err)
exit(err.code)
except KeyboardInterrupt:
LOG.error('** KeyboardInterrupt **')
exit(1)
except BaseException as err:
LOG.error(error=err)
exit(1)
def _get_remote_tag(self,
tag: str,
remote: str,
depth: int = 0,
remove_dir: bool = False) -> Optional[Tag]:
"""Returns Tag object
:param str tag: Tag name
:param str remote: Remote name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool remove_dir: Whether to remove the directory if commands fail
:return: GitPython Tag object if it exists, otherwise None
"""
self._remote(remote, remove_dir=remove_dir)
self.fetch(remote,
depth=depth,
ref=GitRef(tag=tag),
remove_dir=remove_dir)
try:
return self.repo.tags[tag]
except (GitError, IndexError) as err:
LOG.error(f'No existing tag {fmt.ref(tag)}')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
LOG.debug(error=err)
return None
except BaseException:
LOG.error('Failed to get tag')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
def prune_branch_local(self, branch: str, force: bool) -> None:
"""Prune local branch
:param str branch: Branch name to delete
:param bool force: Force delete branch
"""
if branch not in self.repo.heads:
CONSOLE.stdout(f" - Local branch {fmt.ref(branch)} doesn't exist")
return
prune_branch = self.repo.heads[branch]
if self.repo.head.ref == prune_branch:
try:
CONSOLE.stdout(f' - Checkout ref {fmt.ref(self.default_ref.short_ref)}')
self.repo.git.checkout(self.default_ref.short_ref)
except GitError:
LOG.error(f'Failed to checkout ref {fmt.ref(self.default_ref.short_ref)}')
raise
try:
CONSOLE.stdout(f' - Delete local branch {fmt.ref(branch)}')
self.repo.delete_head(branch, force=force)
except GitError:
LOG.error(f'Failed to delete local branch {fmt.ref(branch)}')
raise
def _create_branch_local_tracking(self,
branch: str,
remote: str,
depth: int,
fetch: bool = True,
remove_dir: bool = False) -> None:
"""Create and checkout tracking branch
:param str branch: Branch name
:param str remote: Remote name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool fetch: Whether to fetch before creating branch
:param bool remove_dir: Whether to remove the directory if commands fail
"""
origin = self._remote(remote, remove_dir=remove_dir)
if fetch:
self.fetch(remote,
depth=depth,
ref=GitRef(branch=branch),
remove_dir=remove_dir)
try:
CONSOLE.stdout(f' - Create branch {fmt.ref(branch)}')
self.repo.create_head(branch, origin.refs[branch])
except BaseException:
LOG.error(f'Failed to create branch {fmt.ref(branch)}')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
else:
self._set_tracking_branch(remote, branch, remove_dir=remove_dir)
self._checkout_branch_local(branch, remove_dir=remove_dir)
def _checkout_new_repo_tag(self,
tag: str,
remote: str,
depth: int,
remove_dir: bool = False) -> None:
"""Checkout tag or fail and delete repo if it doesn't exist
:param str tag: Tag name
:param str remote: Remote name
:param int depth: Git clone depth. 0 indicates full clone, otherwise must be a positive integer
:param bool remove_dir: Whether to remove the directory if commands fail
"""
remote_tag = self._get_remote_tag(tag,
remote,
depth=depth,
remove_dir=remove_dir)
if remote_tag is None:
return
try:
CONSOLE.stdout(f' - Checkout tag {fmt.ref(tag)}')
self.repo.git.checkout(remote_tag)
except BaseException:
LOG.error(f'Failed to checkout tag {fmt.ref(tag)}')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
def current_timestamp(self) -> str:
"""Current timestamp of HEAD commit"""
try:
return self.repo.git.log('-1', '--format=%cI')
except GitError:
LOG.error('Failed to find current timestamp')
raise
def install_lfs_hooks(self) -> None:
"""Install git lfs hooks"""
CONSOLE.stdout(" - Update git lfs hooks")
try:
self.repo.git.lfs('install', '--local')
except GitError:
LOG.error('Failed to update git lfs hooks')
raise
def pull(self) -> None:
"""Pull upstream changes"""
try:
CONSOLE.stdout(' - Pull latest changes')
CONSOLE.stdout(self.repo.git.pull())
except GitError:
LOG.error('Failed to pull latest changes')
raise
def pull_lfs(self) -> None:
"""Pull lfs files"""
try:
CONSOLE.stdout(' - Pull git lfs files')
self.repo.git.lfs('pull')
except GitError:
LOG.error('Failed to pull git lfs files')
raise
def push(self) -> None:
"""Push changes"""
try:
CONSOLE.stdout(' - Push local changes')
CONSOLE.stdout(self.repo.git.push())
except GitError:
LOG.error('Failed to push local changes')
raise
def _abort_rebase(self) -> None:
"""Abort rebase"""
if not self._is_rebase_in_progress:
return
try:
self.repo.git.rebase('--abort')
except GitError:
LOG.error('Failed to abort rebase')
raise
def _clean(self, args: str) -> None:
"""Clean git directory
:param str args: Git clean args
"""
try:
self.repo.git.clean(args)
except GitError:
LOG.error('Failed to clean git repo')
raise
def _create_branch_local(self, branch: str) -> None:
"""Create local branch
:param str branch: Branch name
"""
try:
CONSOLE.stdout(f' - Create branch {fmt.ref(branch)}')
self.repo.create_head(branch)
except GitError:
LOG.error(f'Failed to create branch {fmt.ref(branch)}')
raise
def commit(self, message: str) -> None:
"""Commit current changes
:param str message: Git commit message
"""
try:
CONSOLE.stdout(' - Commit current changes')
CONSOLE.stdout(self.repo.git.commit(message=message))
except GitError:
LOG.error('Failed to commit current changes')
raise
def status_verbose(self) -> None:
"""Print git status
Equivalent to: ``git status -vv``
"""
command = 'git status -vv'
CONSOLE.stdout(fmt.command(command))
try:
execute_command(command, self.repo_path)
except CalledProcessError:
LOG.error('Failed to print verbose status')
raise
def _create_repo(self) -> Repo:
"""Create Repo instance for self.repo_path
:return: GitPython Repo instance
"""
try:
return Repo(self.repo_path)
except GitError:
LOG.error(
f"Failed to create Repo instance for {fmt.path(self.repo_path)}"
)
raise
def _print_has_remote_branch_message(self, branch: str) -> None:
"""Print output message for existing remote branch
:param str branch: Branch name
"""
try:
self.repo.git.config('--get', 'branch.' + branch + '.merge')
CONSOLE.stdout(
f' - Tracking branch {fmt.ref(branch)} already exists')
except GitError:
LOG.error(f'Remote branch {fmt.ref(branch)} already exists')
raise
def remove_directory(dir_path: Path, check: bool = True) -> None:
"""Remove directory at path
:param str dir_path: Path to directory to remove
:param bool check: Whether to raise errors
"""
try:
shutil.rmtree(dir_path)
except shutil.Error:
LOG.error(f"Failed to remove directory {fmt.path(dir_path)}")
if check:
raise
def _print_yaml(yaml_file: Path) -> None:
"""Private print current clowder yaml file
:param Path yaml_file: Path to yaml file
"""
try:
with yaml_file.open() as raw_file:
contents = raw_file.read()
CONSOLE.stdout(contents.rstrip())
except IOError:
LOG.error(f"Failed to open file '{yaml_file}'")
raise
def _find_rev_by_timestamp(self, timestamp: str, ref: str) -> str:
"""Find rev by timestamp
:param str timestamp: Commit ref timestamp
:param str ref: Reference ref
:return: Commit sha at or before timestamp
"""
try:
return self.repo.git.log('-1', '--format=%H',
'--before=' + timestamp, ref)
except GitError:
LOG.error('Failed to find revision from timestamp')
raise
def _is_tracking_branch(self, branch: str) -> bool:
"""Check if branch is a tracking branch
:param str branch: Branch name
:return: True, if branch has a tracking relationship
"""
try:
local_branch = self.repo.heads[branch]
tracking_branch = local_branch.tracking_branch()
return True if tracking_branch else False
except GitError:
LOG.error(f'No existing branch {fmt.ref(branch)}')
raise
def _pull(self, remote: str, branch: str) -> None:
"""Pull from remote branch
:param str remote: Remote name
:param str branch: Branch name
"""
CONSOLE.stdout(f' - Pull from {fmt.remote(remote)} {fmt.ref(branch)}')
try:
execute_command(f"git pull {remote} {branch}", self.repo_path)
except CalledProcessError:
LOG.error(
f'Failed to pull from {fmt.remote(remote)} {fmt.ref(branch)}')
raise
def git_config_add_local(self, variable: str, value: str) -> None:
"""Add local git config value for given variable key
:param str variable: Fully qualified git config variable
:param str value: Git config value
"""
try:
self.repo.git.config('--local', '--add', variable, value)
except GitError:
LOG.error(
f'Failed to add local git config value {value} for variable {variable}'
)
raise
def validate_yaml_file(parsed_yaml: dict, file_path: Path) -> None:
"""Validate yaml file
:param dict parsed_yaml: Parsed yaml dictionary
:param Path file_path: Path to yaml file
"""
json_schema = _load_json_schema(file_path.stem)
try:
jsonschema.validate(parsed_yaml, json_schema)
except jsonschema.exceptions.ValidationError:
LOG.error(
f'Yaml json schema validation failed {fmt.invalid_yaml(file_path.name)}\n'
)
raise
def add(self, files: str) -> None:
"""Add files to git index
:param str files: Files to git add
:raise:
"""
CONSOLE.stdout(' - Add files to git index')
try:
CONSOLE.stdout(self.repo.git.add(files))
except GitError:
LOG.error("Failed to add files to git index")
raise
else:
self.status_verbose()
def _remote(self, remote: str, remove_dir: bool = False) -> Remote:
"""Get GitPython Remote instance
:param str remote: Remote name
:param bool remove_dir: Whether to remove the directory if commands fail
:return: GitPython Remote instance
"""
try:
return self.repo.remotes[remote]
except GitError:
LOG.error(f'No existing remote {fmt.remote(remote)}')
if remove_dir:
remove_directory(self.repo_path, check=False)
raise
def _checkout_sha(self, sha: str) -> None:
"""Checkout commit by sha
:param str sha: Commit sha
"""
try:
if self.repo.head.commit.hexsha == sha:
CONSOLE.stdout(' - On correct commit')
return
CONSOLE.stdout(f' - Checkout commit {fmt.ref(sha)}')
self.repo.git.checkout(sha)
except GitError:
LOG.error(f'Failed to checkout commit {fmt.ref(sha)}')
raise
def yaml_string(yaml_output: dict) -> str:
"""Return yaml string from python data structures
:param dict yaml_output: YAML python object
:return: YAML as a string
"""
try:
return pyyaml.safe_dump(yaml_output,
default_flow_style=False,
indent=2,
sort_keys=False)
except pyyaml.YAMLError:
LOG.error(f"Failed to dump yaml file contents", )
raise