def deploy(repo_name):
for s in config:
if s['repo_name'] == repo_name:
git_repo = GitRepo(remote_url=s['repo_url'],
local_folder=s['repo_name'])
if s['deploy_type'] == 's3':
server = S3Bucket(s['aws_key_id'], s['aws_key'],
s['s3_bucket'])
prev_hash = server.get_value(VERSION_FILE)
if '' == prev_hash:
files_to_upload = git_repo.all_files()
files_to_delete = []
else:
files_to_upload, files_to_delete = \
git_repo.changed_files(prev_hash)
server.upload_files(files_to_upload, all_public=True)
server.delete_files(files_to_delete)
server.set_value(VERSION_FILE, git_repo.head_hash())
elif s['deploy_type'] == 'ssh':
print 'did nothing'
else:
assert False, \
'Wrong deploy type: %s. Only support S3 and SSH now' \
% s['deploy_type']
return
def deploy(repo_name):
for s in config:
if s['repo_name'] == repo_name:
git_repo = GitRepo(
remote_url=s['repo_url'],
local_folder=s['repo_name'])
if s['deploy_type'] == 's3':
server = S3Bucket(
s['aws_key_id'],
s['aws_key'],
s['s3_bucket'])
prev_hash = server.get_value(VERSION_FILE)
if '' == prev_hash:
files_to_upload = git_repo.all_files()
files_to_delete = []
else:
files_to_upload, files_to_delete = \
git_repo.changed_files(prev_hash)
server.upload_files(files_to_upload, all_public=True)
server.delete_files(files_to_delete)
server.set_value(VERSION_FILE, git_repo.head_hash())
elif s['deploy_type'] == 'ssh':
print 'did nothing'
else:
assert False, \
'Wrong deploy type: %s. Only support S3 and SSH now' \
% s['deploy_type']
return
def _clone_git(self, git_url):
logger.debug('Cloning/pulling repository %s.', self.name)
try:
if os.path.exists(self.name):
try:
GitRepo(self.name).git.reset('--hard', 'origin')
GitRepo(self.name).git.pull()
logger.debug('Repository %s pulled.', self.name)
except Exception:
logger.error('Unable to pull invalid repository %s.',
self.name)
return generic_error(msg='Error updating Git Repo! ' +
'Please ensure the path ' +
'is a valid Git Repo.')
else:
os.makedirs(self.name)
GitRepo.clone_from(git_url, self.name)
logger.debug('Repository %s cloned.', self.name)
except Exception:
logger.error('Unable to access repository %s.', self.name)
return generic_error(msg='Error cloning/pulling Git Repo! ' +
'Please ensure you have access.')
def get_git_repo(self, repo_url, local_repo):
""" Get a git repository.
Parameters
----------
repo_url: str
The location of the git repository (an url if local is False, a
local path otherwise)
local_repo: bool
If True, get the repository from a local directory instead of the
web
Returns
-------
str
The temporary path to which the repository has been copied
GitRepo
The repository object
Raises
------
FileNotFoundError
If repo_url is not an existing directory
git.InvalidGitRepositoryError
If the directory in repo_url is not a git repository
git.GitCommandError
If the url in repo_url is not a git repository, or access to the
repository is denied
"""
project_path = tempfile.mkdtemp()
if local_repo:
project_path = os.path.join(tempfile.mkdtemp(), 'repo')
try:
shutil.copytree(repo_url, project_path)
repo = GitRepo(project_path)
except FileNotFoundError as e:
shutil.rmtree(project_path)
raise e
except InvalidGitRepositoryError as e:
shutil.rmtree(project_path)
raise InvalidGitRepositoryError(
f"\"{repo_url}\" is not a local git repository.") from e
else:
try:
GitRepo.clone_from(repo_url, project_path)
repo = GitRepo(project_path)
except GitCommandError as e:
shutil.rmtree(project_path)
raise e
return project_path, repo
def download(cls,
manifest=None,
localdir=None,
version=None,
override_existing=False,
**kwargs):
try:
repo = GitRepo(localdir)
except (InvalidGitRepositoryError, NoSuchPathError):
repo = GitRepo.init(localdir)
if manifest.git is not None:
try:
repo.create_remote('origin', manifest.git)
except GitCommandError as e:
pass
try:
if sys.stdout.isatty():
repo.remotes.origin.fetch(progress=GitProgressBar(
label="%s@%s" % (str(manifest), version.version)))
else:
for fetch_info in repo.remotes.origin.fetch():
logger.debug(
"Updated %s %s to %s" %
(manifest.git, fetch_info.ref, fetch_info.commit))
# self.last_checked = datetime.now()
except (GitCommandError, AttributeError) as e:
logger.error("Could not fetch %s: %s" % (manifest.git, str(e)))
if version.git_sha is not None:
repo.git.checkout(version.git_sha)
def get_env(self, directory=None):
"""Determine environment name."""
if self.environment_override_name in self.env_vars:
return self.env_vars[self.environment_override_name]
if self.runway_config.get('ignore_git_branch', False):
LOGGER.info('Skipping environment lookup from current git branch '
'("ignore_git_branch" is set to true in the runway '
'config)')
else:
# These are not located with the top imports because they throw an
# error if git isn't installed
from git import Repo as GitRepo
from git.exc import InvalidGitRepositoryError
if directory is None:
directory = self.module_root
try:
b_name = GitRepo(
directory,
search_parent_directories=True
).active_branch.name
LOGGER.info('Deriving environment name from git branch %s...',
b_name)
return self.get_env_from_branch(b_name)
except InvalidGitRepositoryError:
pass
LOGGER.info('Deriving environment name from directory %s...',
self.env_root)
return self.get_env_from_directory(os.path.basename(self.env_root))
def get_env(path, ignore_git_branch=False):
"""Determine environment name."""
if 'DEPLOY_ENVIRONMENT' in os.environ:
return os.environ['DEPLOY_ENVIRONMENT']
if ignore_git_branch:
LOGGER.info('Skipping environment lookup from current git branch '
'("ignore_git_branch" is set to true in the Runway '
'config)')
else:
# These are not located with the top imports because they throw an
# error if git isn't installed
from git import Repo as GitRepo
from git.exc import InvalidGitRepositoryError
try:
b_name = GitRepo(path,
search_parent_directories=True).active_branch.name
LOGGER.info('Deriving environment name from git branch %s...',
b_name)
return get_env_from_branch(b_name)
except InvalidGitRepositoryError:
pass
except TypeError:
LOGGER.warning('Unable to retrieve the current git branch name!')
LOGGER.warning('Typically this occurs when operating in a '
'detached-head state (e.g. what Jenkins uses when '
'checking out a git branch). Set the '
'DEPLOY_ENVIRONMENT environment variable to the '
'name of the logical environment (e.g. "export '
'DEPLOY_ENVIRONMENT=dev") to bypass this error.')
sys.exit(1)
LOGGER.info('Deriving environment name from directory %s...', path)
return get_env_from_directory(os.path.basename(path))
def get_env(path, ignore_git_branch=False):
"""Determine environment name."""
if 'DEPLOY_ENVIRONMENT' in os.environ:
return os.environ['DEPLOY_ENVIRONMENT']
if ignore_git_branch:
LOGGER.info('Skipping environment lookup from current git branch '
'("ignore_git_branch" is set to true in the runway '
'config)')
else:
# These are not located with the top imports because they throw an
# error if git isn't installed
from git import Repo as GitRepo
from git.exc import InvalidGitRepositoryError
try:
b_name = GitRepo(path,
search_parent_directories=True).active_branch.name
LOGGER.info('Deriving environment name from git branch %s...',
b_name)
return get_env_from_branch(b_name)
except InvalidGitRepositoryError:
pass
LOGGER.info('Deriving environment name from directory %s...', path)
return get_env_from_directory(os.path.basename(path))
def checkout(ctx, self, version=None, retry=False):
"""Checkout a version of the repository."""
if ctx.dont_checkout:
return
if version is None:
version = self.version
if self._type == RepositoryType.ARCH:
return
elif self._type == RepositoryType.PLAT and self._source == UNIKRAFT_CORE:
return
elif version is not None:
try:
repo = GitRepo(self._localdir)
except (NoSuchPathError, InvalidGitRepositoryError):
logger.debug("Attempting to checkout %s before update!" % self)
# Allow one retry
if retry is False:
self.update()
self.checkout(version, True)
return
try:
# If this throws an exception, it means we have never checked
# out the repository before.
commit_hash = str(repo.head.commit)
# Determine if the repository has already been checked out at
# this version
if commit_hash.startswith(version) \
or version in self._known_versions.keys() and self._known_versions[version] == commit_hash:
logger.debug("%s already at %s" % (self._name, version))
return
except ValueError as e:
pass
logger.debug("Checking-out %s@%s..." % (self._name, version))
# First simply attempting what was specified
try:
repo.git.checkout(version)
except GitCommandError as e1:
# Don't try well-known branches with well-known RELEASE prefix:
if version != BRANCH_MASTER and version != BRANCH_STAGING:
try:
repo.git.checkout('RELEASE-%s' % version)
except GitCommandError as e2:
if not ctx.ignore_checkout_errors:
logger.error("Could not checkout %s@%s: %s" %
(self._name, version, str(e2)))
sys.exit(1)
elif not ctx.ignore_checkout_errors:
logger.error("Could not checkout %s@%s: %s" %
(self._name, version, str(e1)))
sys.exit(1)
def __init__(self, version: str):
from git import Repo as GitRepo
from svn.remote import RemoteClient as SvnRepo
self.git_repo = None
self.svn_repo = None
if version is BLENDER_VERSION_MASTER:
self.git_repo = GitRepo(Blender.GIT_BASE_URL)
self.svn_repo = SvnRepo(os.path.join(Blender.SVN_BASE_URL,
Blender.SVN_MASTER))
else:
self.match = BLENDER_VERSION_REGEX.match(version)
if self.match:
matching_git_tags = [tag for tag in GitRepo(Blender.GIT_BASE_URL).tags
if tag == self.match.group(0)]
self.git_repo = matching_git_tags[0] if matching_git_tags else None
if self.git_repo:
pass
else:
raise Exception(f"Blender {version} does not exist in git")
svn_version_tag = (f"blender-{self.match.group(1)}"
f"{self.match.group(2) if not self.match.group(2).startswith("-rc")}-release")
blender_svn_tags = os.path.join(Blender.SVN_BASE_URL,
Blender.SVN_TAGS)
matching_svn_tags = [os.path.join(blender_svn_tags, tag) for
tag in
SvnRepo(os.path.join(Blender.SVN_BASE_URL,
Blender.SVN_TAGS)).list() if
tag == svn_version_tag]
self.svn_repo = SvnRepo(matching_svn_tags[0]) if matching_svn_tags else None
def __init__(self, full_name):
self.full_name = full_name
try:
self.repo = GitRepo('repos/{}'.format(full_name))
except NoSuchPathError:
self.repo = GitRepo.clone_from(
'[email protected]:{}.git'.format(full_name),
'repos/{}'.format(full_name))
def meta_repo():
"""Get a reference to the meta repo that contains the templates and
lists of students
Returns:
git.Repo: metadata repo
"""
config = Config.load_config()
return GitRepo(config.meta_path)
def clone_or_pull(self):
try:
if os.path.isdir(self.clone_dir):
git_repo = GitRepo(self.clone_dir)
git_repo.remotes.origin.pull("refs/heads/master:refs/heads/origin")
else:
git_repo = GitRepo.clone_from(self.git_url, self.clone_dir)
except GitError:
raise exceptions.GitException(self, "Cannot clone or pull Git repository")
self.git_repo = git_repo
def git_initialise():
try:
# attempt to open existing repository
repository = GitRepo(GIT_REPO_PATH)
print(f"Opened existing repository in '{GIT_REPO_PATH}'")
except (InvalidGitRepositoryError, NoSuchPathError):
# initialise bare repository
os.makedirs(GIT_REPO_PATH, exist_ok=True)
repository = GitRepo.init(GIT_REPO_PATH)
print(f"Initialised new repository in '{GIT_REPO_PATH}'")
return repository
def checkout(self, full_path: pathlib.Path):
"""Retrieve Blender code from Git
Keyword Arguments:
full_path {pathlib.Path} -- place to clone code to
"""
try:
repo = GitRepo(str(full_path))
except:
GitRepo.clone_from(self.BASE_URL, str(full_path))
repo = GitRepo(str(full_path))
repo.heads.master.checkout()
repo.remotes.origin.pull()
if self.tag is not None:
repo.git.checkout(self.tag)
repo.git.submodule('update', '--init', '--recursive')
for submodule in repo.submodules:
submodule_repo = submodule.module()
submodule_repo.heads.master.checkout()
submodule_repo.remotes.origin.pull()
if self.tag is not None:
try:
submodule_repo.git.checkout(self.tag)
except:
pass
def __init__(self, repo_path: str):
"""
Initialize GitWrapper. Should be initialized by ZenML Repository.
Args:
repo_path:
Raises:
InvalidGitRepositoryError: If repository is not a git repository.
NoSuchPathError: If the repo_path does not exist.
"""
# TODO [ENG-163]: Raise ZenML exceptions here instead.
self.repo_path: str = repo_path
self.git_root_path: str = os.path.join(repo_path, GIT_FOLDER_NAME)
self.git_repo = GitRepo(self.repo_path)
def get_env(path, ignore_git_branch=False, prompt_if_unexpected=False):
"""Determine environment name.
Args:
path (str): Path to check for deploy environment name.
ignore_git_branch (bool): Skip checking for git branch name.
(*default:* ``False``)
prompt_if_unexpected (bool): If the branch name is an unexpected
format/value, the user will be prompted if they would like to
enter a different deploy environment. (*default:* ``False``)
Returns:
str: Deploy environment.
"""
if 'DEPLOY_ENVIRONMENT' in os.environ:
return os.environ['DEPLOY_ENVIRONMENT']
if ignore_git_branch:
LOGGER.info('Skipping environment lookup from current git branch '
'("ignore_git_branch" is set to true in the Runway '
'config)')
else:
# These are not located with the top imports because they throw an
# error if git isn't installed
from git import Repo as GitRepo # pylint: disable=import-outside-toplevel
from git.exc import InvalidGitRepositoryError # pylint: disable=import-outside-toplevel
try:
b_name = GitRepo(
path,
search_parent_directories=True
).active_branch.name
LOGGER.info('Deriving environment name from git branch %s...',
b_name)
return get_env_from_branch(b_name, prompt_if_unexpected)
except InvalidGitRepositoryError:
pass
except TypeError:
LOGGER.warning('Unable to retrieve the current git branch name!')
LOGGER.warning('Typically this occurs when operating in a '
'detached-head state (e.g. what Jenkins uses when '
'checking out a git branch). Set the '
'DEPLOY_ENVIRONMENT environment variable to the '
'name of the logical environment (e.g. "export '
'DEPLOY_ENVIRONMENT=dev") to bypass this error.')
sys.exit(1)
LOGGER.info('Deriving environment name from directory %s...', path)
return get_env_from_directory(os.path.basename(path))
def get_git_repo(repo_path, init=False):
if os.path.isdir(repo_path):
try:
return GitRepo(repo_path)
except InvalidGitRepositoryError:
if init:
return GitRepo.init(repo_path)
elif init:
try:
create_path(repo_path)
return get_git_repo(repo_path, init=init)
except FileNotFoundError:
pass
raise ValueError('Could not create a repo on this path {}'.format(repo_path))
def __attrs_post_init__(self):
"""Initialize computed attributes."""
#: Configure Renku path.
path = Path(self.renku_home)
if not path.is_absolute():
path = self.path / path
path.relative_to(path)
self.renku_path = path
#: Create an instance of a Git repository for the given path.
try:
self.git = GitRepo(str(self.path))
except InvalidGitRepositoryError:
self.git = None
def get_commit_hash(self):
logger.debug('Getting commit hash for repository %s.', self.name)
if not os.path.exists(self.name):
logger.error('When getting commit hash, path of %s not found.',
self.name)
return no_code_dir_error()
repo = GitRepo(self.name)
sha = repo.head.object.hexsha
logger.debug('Commit hash for repository %s obtained.', self.name)
return {"sha": sha}
def clone_student_repo(student, dest=None):
"""Clones a student repo into a temporary directory
Arguments:
student (Student)
Returns:
git.Repo: repo at the cloned path
"""
logger.info('Cloning repo: %s', student.github)
if dest is None:
dest = tempfile.mkdtemp()
repo_url = student.repo_url
logger.debug('%s -> %s', repo_url, dest)
GitRepo.clone_from(repo_url, dest)
return GitRepo(dest)
def get(self, request, *args, **kwargs):
if 'remove' in request.GET.keys():
self.get_object().delete()
return redirect(self.success_url)
elif 'generate' in request.GET.keys():
os.system(
f"pycco repos/{self.get_object().relative_dir}/**/*.py -p")
#pass
elif 'update' in request.GET.keys():
repo = GitRepo(
os.path.join(settings.REPOS_DIR,
self.get_object().relative_dir))
o = repo.remotes.origin
o.pull()
return super().get(request, *args, **kwargs)
def __init__(self, repo_path: Text):
"""
Initialize GitWrapper. Should be initialize by ZenML Repository.
Args:
repo_path:
Raises:
InvalidGitRepositoryError: If repository is not a git repository.
NoSuchPathError: If the repo_path does not exist.
"""
# TODO: [LOW] Raise ZenML exceptions here instead.
self.repo_path: Text = repo_path
self.git_root_path: Text = os.path.join(repo_path, GIT_FOLDER_NAME)
self.git_repo: GitRepo = GitRepo(self.repo_path)
# TODO: [LOW] Check whether this is necessary.
assert not self.git_repo.bare
def clone_or_pull(self):
if not self.remote_available():
raise exceptions.GitException(
self, "Cannot clone or pull Git repository")
try:
if os.path.isdir(self.clone_dir):
git_repo = GitRepo(self.clone_dir)
git_repo.remotes.origin.fetch(tags=True, force=True)
git_repo.git.reset("--hard", "origin/master")
else:
git_repo = GitRepo.clone_from(self.git_url, self.clone_dir)
except GitError:
raise exceptions.GitException(
self, "Cannot clone or pull Git repository")
self.git_repo = git_repo
def __init__(self, git_url, email, schema_type):
super(Repo, self).__init__()
parsed_git = giturlparse.parse(git_url)
self.git_url = git_url
self.owner = self.find_owner(parsed_git)
self.name = parsed_git.name
self.email = email
if os.path.isdir(self.clone_dir):
self.git_repo = GitRepo(self.clone_dir)
else:
self.git_repo = None
self.current_tag = None
self.cache_latest_valid_tag = None
if schema_type not in self.SCHEMA_TYPES:
raise exceptions.InvalidSchemaTypeException(
self,
"`%s` is not a supported schema type. Supported: %s" %
(schema_type, ",".join(self.SCHEMA_TYPES)),
)
self.schema_type = schema_type
def update(self):
"""Update this particular repository."""
repo = None
try:
repo = GitRepo(self._localdir)
# Not a repository? No problem, let's clone it:
except (InvalidGitRepositoryError, NoSuchPathError) as e:
repo = GitRepo.init(self._localdir)
repo.create_remote('origin', self._source)
try:
for fetch_info in repo.remotes.origin.fetch():
logger.debug("Updated %s %s to %s" %
(self._source, fetch_info.ref, fetch_info.commit))
self._last_checked = datetime.now()
except (GitCommandError, AttributeError) as e:
logger.error("Could not fetch %s: %s" % (self._source, str(e)))
def is_type(cls, origin_url=None):
if origin_url is None:
return False
try:
if origin_url.startswith("file://"):
origin_url = origin_url[7:]
GitRepo(origin_url, search_parent_directories=True)
return True
except Exception:
pass
try:
GitCmd().ls_remote(origin_url)
return True
except Exception:
pass
return False
def is_type(cls, source=None):
if source is None:
return False
try:
if source.startswith("file://"):
source = source[7:]
GitRepo(source, search_parent_directories=True)
return True
except Exception:
pass
try:
GitCmd().ls_remote(source)
return True
except Exception:
pass
return False
def check_create_local_repo(config) -> GitRepo:
"""Create a local repo for saving state and push it to
the user's personal ref. Checkout any existing version
on the user's personal remote, or create a new commit"""
path = Path(config.cwd, "_state")
if not path.exists():
os.mkdir(path)
try:
repo = GitRepo(path)
if "origin" not in [r.name
for r in repo.remotes] and config._state_ref:
repo.create_remote(
'origin',
f"ssh://{config.GERRIT_HOST[8:]}/{config.GERRIT_STATE_PATH}")
except exc.InvalidGitRepositoryError:
repo = GitRepo.init(path)
if config._state_ref:
repo.create_remote(
'origin',
f"ssh://{config.GERRIT_HOST[8:]}/{config.GERRIT_STATE_PATH}")
fetch_and_checkout(config, repo)
state_path = Path(repo.working_tree_dir, "state.bin")
if not state_path.exists():
with open(state_path, 'wb') as state_file:
pickle.dump({}, state_file)
repo.index.add('state.bin')
repo.index.commit("Empty state")
if config._state_ref:
repo.remotes.origin.push(['-f', f"HEAD:{config._state_ref}"])
if not config._state_ref:
print(
"\nWARN: Unable to create git remote for state!\n"
"WARN: State will only be saved locally to _state/state.bin.\n"
"INFO: Please configure an ssh user in ~/.ssh/config for your gerrit host\n"
"INFO: as set by 'GERRIT_HOST' in config.yaml in order to save state in gerrit.\n"
)
return repo
def check_repo(self, repo_url, git_token=None, local_repo=False):
"""
Check git token validity for the repository
Parameters
----------
repo_url: str
The location of a git repository (an url if local_repo is False, a
local path otherwise)
git_token: str, optional
Git personal access token to authenticate to the git server
local_repo: bool, optional
If True, get the repository from a local directory instead of the
web
Returns
-------
bool
True if the git token is valid for the repository, False otherwise
"""
if local_repo:
try:
GitRepo(repo_url)
except InvalidGitRepositoryError:
return False, 'InvalidGitRepositoryError'
except NoSuchPathError:
return False, 'NoSuchPathError'
else:
g = git.cmd.Git()
if git_token is not None and len(git_token) > 0:
repo_url = repo_url.replace('https://',
f'https://*****:*****@')
try:
g.ls_remote(repo_url)
except GitCommandError:
return False, 'GitCommandError'
return True, None
def __init__(self,
name,
repopath,
secret,
sshkey='',
action='default',
branch='master',
script=None):
self.title = name + branch
self.name = name
self.repo = GitRepo(repopath)
action = action.lower()
if action in ['default', 'script']:
self.action = action
else:
raise ValueError('action for {} is not supported'.format(name))
if action == 'script':
if script is None:
raise ValueError('Script not specified for {}'.format(name))
elif not app.config['TESTING'] and not isabs(script):
raise ValueError(
'Script for {} is not a absolute path'.format(name))
self.script = script
if sshkey != '' and isfile(sshkey):
self.ssh_cmd = 'ssh -i {}'.format(sshkey)
else:
self.ssh_cmd = 'ssh'
self.branch = branch
self.repo.git.checkout('-B', self.branch)
self.__update()
self.secret = secret
def __init__(self, config):
self.config = config
self.repo = GitRepo.factory(self.config.general_options["git_repo"])
self.log_dir = self.config.general_options["git_repo"] + "/balog"
if not os.path.isdir(self.log_dir):
os.mkdir(self.log_dir)