def add_remote(git_repo):
# add origin remote
output, errors = AgentGitHandler.execute_git_command(
["remote", "add", "origin", git_repo.repo_url],
git_repo.local_repo_path)
if len(output) > 0:
raise GitRepositorySynchronizationException(
"Error in adding remote origin %s for local repository %s" %
(git_repo.repo_url, git_repo.local_repo_path))
# fetch
output, errors = AgentGitHandler.execute_git_command(
["fetch"], git_repo.local_repo_path)
if "Resolving deltas: 100%" not in output:
raise GitRepositorySynchronizationException(
"Error in fetching from remote origin %s for local repository %s"
% (git_repo.repo_url, git_repo.local_repo_path))
# checkout master
output, errors = AgentGitHandler.execute_git_command(
["checkout", "master"], git_repo.local_repo_path)
if "Branch master set up to track remote branch master from origin." not in output:
raise GitRepositorySynchronizationException(
"Error in checking out master branch %s for local repository %s"
% (git_repo.repo_url, git_repo.local_repo_path))
return True
def pull(git_repo):
# git reset to make sure no uncommitted changes are present before the pull, no conflicts will occur
AgentGitHandler.execute_git_command(["reset", "--hard"],
git_repo.local_repo_path)
# HEAD before pull
(init_head, init_errors) = AgentGitHandler.execute_git_command(
["rev-parse", "HEAD"], git_repo.local_repo_path)
try:
repo = Repo(git_repo.local_repo_path)
repo.remotes.origin.pull()
if repo.is_dirty():
raise GitRepositorySynchronizationException(
"Git pull operation left the repository in dirty state")
except (GitCommandError, GitRepositorySynchronizationException) as e:
raise GitRepositorySynchronizationException(
"Git pull operation on %s for tenant %s failed: %s" %
(git_repo.repo_url, git_repo.tenant_id, e))
# HEAD after pull
(end_head, end_errors) = AgentGitHandler.execute_git_command(
["rev-parse", "HEAD"], git_repo.local_repo_path)
# check if HEAD was updated
if init_head != end_head:
AgentGitHandler.log.debug(
"Artifacts were updated as a result of the pull operation, thread: %s - %s"
% (current_thread().getName(), current_thread().ident))
return True
else:
return False
def retry_clone(git_repo):
"""Retry 'git clone' operation for defined number of attempts with defined intervals
"""
git_clone_successful = False
# Read properties from agent.conf
max_retry_attempts = int(Config.artifact_clone_retry_count)
retry_interval = int(Config.artifact_clone_retry_interval)
retry_attempts = 0
# Iterate until git clone is successful or reaches max retry attempts
while git_clone_successful is False and retry_attempts < max_retry_attempts:
try:
retry_attempts += 1
AgentGitHandler.clone(git_repo)
AgentGitHandler.log.info(
"Retrying attempt to git clone operation for tenant %s successful"
% git_repo.tenant_id)
git_clone_successful = True
except GitRepositorySynchronizationException as e:
AgentGitHandler.log.exception(
"Retrying git clone attempt %s failed: %s" %
(retry_attempts, e))
if retry_attempts < max_retry_attempts:
time.sleep(retry_interval)
else:
raise GitRepositorySynchronizationException(
"All attempts failed while retrying git clone: %s" % e)
def clone(git_repo):
try:
# create a temporary location to clone
temp_repo_path = os.path.join(tempfile.gettempdir(),
"pca_temp_" + git_repo.tenant_id)
if os.path.isdir(
temp_repo_path) and os.listdir(temp_repo_path) != []:
GitUtils.delete_folder_tree(temp_repo_path)
GitUtils.create_dir(temp_repo_path)
# clone the repo to a temporary location first to avoid conflicts
AgentGitHandler.log.debug(
"Cloning artifacts from URL: %s to temp location: %s" %
(git_repo.repo_url, temp_repo_path))
Repo.clone_from(git_repo.auth_url, temp_repo_path)
# clear the paths to get rid of the following bug in distutils:
# http://stackoverflow.com/questions/9160227/dir-util-copy-tree-fails-after-shutil-rmtree
distutils.dir_util._path_created = {}
# move the cloned dir to application path
copy_tree(temp_repo_path, git_repo.local_repo_path)
AgentGitHandler.log.info(
"Git clone operation for tenant %s successful" %
git_repo.tenant_id)
return git_repo
except GitCommandError as e:
raise GitRepositorySynchronizationException(
"Error while cloning repository for tenant %s: %s" %
(git_repo.tenant_id, e))
def clone(git_repo):
try:
# create a temporary location to clone
temp_repo_path = os.path.join(tempfile.gettempdir(),
"pca_temp_" + git_repo.tenant_id)
if os.path.isdir(
temp_repo_path) and os.listdir(temp_repo_path) != []:
GitUtils.delete_folder_tree(temp_repo_path)
GitUtils.create_dir(temp_repo_path)
# clone the repo to a temporary location first to avoid conflicts
AgentGitHandler.log.debug(
"Cloning artifacts from URL: %s to temp location: %s" %
(git_repo.repo_url, temp_repo_path))
Repo.clone_from(git_repo.auth_url,
temp_repo_path,
branch=git_repo.branch)
# move the cloned dir to application path
copy_tree(temp_repo_path, git_repo.local_repo_path)
AgentGitHandler.log.info(
"Git clone operation for tenant %s successful" %
git_repo.tenant_id)
return git_repo
except GitCommandError as e:
raise GitRepositorySynchronizationException(
"Error while cloning repository for tenant %s: %s" %
(git_repo.tenant_id, e))
def delete_folder_tree(path):
"""
Completely deletes the provided folder
:param str path: Full path of the folder
:return: void
"""
try:
shutil.rmtree(path)
GitUtils.log.debug("Directory [%s] deleted." % path)
except OSError as e:
raise GitRepositorySynchronizationException("Deletion of folder path %s failed: %s" % (path, e))
def create_dir(path):
"""
mkdir the provided path
:param path: The path to the directory to be made
:return: True if mkdir was successful, False if dir already exists
:rtype: bool
"""
try:
os.mkdir(path)
GitUtils.log.debug("Successfully created directory [%s]" % path)
# return True
except OSError as e:
raise GitRepositorySynchronizationException("Directory creating failed in [%s]. " % e)
def clone(git_repo):
if os.path.isdir(git_repo.local_repo_path):
# delete and recreate local repo path if exists
AgentGitHandler.log.debug(
"Local repository path not empty. Cleaning.")
GitUtils.delete_folder_tree(git_repo.local_repo_path)
try:
Repo.clone_from(git_repo.repo_url, git_repo.local_repo_path)
AgentGitHandler.add_repo(git_repo)
AgentGitHandler.log.info(
"Git clone operation for tenant %s successful" %
git_repo.tenant_id)
return git_repo
except GitCommandError as e:
raise GitRepositorySynchronizationException(
"Error while cloning repository: %s" % e)
def retry_clone(git_repo):
"""Retry 'git clone' operation for defined number of attempts with defined intervals
"""
if os.path.isdir(git_repo.local_repo_path) and os.listdir(
git_repo.local_repo_path) != []:
# delete and recreate local repo path if not empty dir
AgentGitHandler.log.debug(
"Local repository path not empty. Cleaning.")
GitUtils.delete_folder_tree(git_repo.local_repo_path)
GitUtils.create_dir(git_repo.local_repo_path)
git_clone_successful = False
# Read properties from agent.conf
max_retry_attempts = int(
Config.read_property(constants.ARTIFACT_CLONE_RETRIES, 5))
retry_interval = int(
Config.read_property(constants.ARTIFACT_CLONE_INTERVAL, 10))
retry_attempts = 0
# Iterate until git clone is successful or reaches max retry attempts
while git_clone_successful is False and retry_attempts < max_retry_attempts:
try:
retry_attempts += 1
Repo.clone_from(git_repo.repo_url, git_repo.local_repo_path)
AgentGitHandler.add_repo(git_repo)
AgentGitHandler.log.info(
"Retrying attempt to git clone operation for tenant %s successful"
% git_repo.tenant_id)
git_clone_successful = True
except GitCommandError as e:
AgentGitHandler.log.warn(
"Retrying git clone attempt %s failed" % retry_attempts)
if retry_attempts < max_retry_attempts:
time.sleep(retry_interval)
pass
else:
raise GitRepositorySynchronizationException(
"Error while retrying git clone: %s" % e)
def push(repo_info):
"""
Commits and pushes new artifacts to the remote repository
:param repo_info:
:return:
"""
git_repo = AgentGitHandler.get_repo(repo_info.tenant_id)
if git_repo is None:
# not cloned yet
raise GitRepositorySynchronizationException(
"Not a valid repository to push from. Aborting")
# Get initial HEAD so in case if push fails it can be reverted to this hash
# This way, commit and push becomes an single operation. No intermediate state will be left behind.
(init_head, init_errors) = AgentGitHandler.execute_git_command(
["rev-parse", "HEAD"], git_repo.local_repo_path)
# stage all untracked files
if AgentGitHandler.stage_all(git_repo.local_repo_path):
AgentGitHandler.log.info(
"Git staged untracked artifacts successfully")
else:
AgentGitHandler.log.info("Git could not stage untracked artifacts")
# check if modified files are present
modified = AgentGitHandler.has_modified_files(git_repo.local_repo_path)
AgentGitHandler.log.debug("[Git] Modified: %s" % str(modified))
if not modified:
AgentGitHandler.log.debug(
"No changes detected in the local repository for tenant %s" %
git_repo.tenant_id)
return
# commit to local repositpory
commit_message = "tenant [%s]'s artifacts committed to local repo at %s" \
% (git_repo.tenant_id, git_repo.local_repo_path)
# TODO: set configuratble names, check if already configured
commit_name = git_repo.tenant_id
commit_email = "*****@*****.**"
# git config
AgentGitHandler.execute_git_command(
["config", "user.email", commit_email], git_repo.local_repo_path)
AgentGitHandler.execute_git_command(
["config", "user.name", commit_name], git_repo.local_repo_path)
# commit
(output, errors) = AgentGitHandler.execute_git_command(
["commit", "-m", commit_message], git_repo.local_repo_path)
if errors.strip() == "":
commit_hash = AgentGitHandler.find_between(output, "[master",
"]").strip()
AgentGitHandler.log.debug(
"Committed artifacts for tenant : %s : %s " %
(git_repo.tenant_id, commit_hash))
else:
AgentGitHandler.log.exception(
"Committing artifacts to local repository failed for tenant: %s, Cause: %s"
% (git_repo.tenant_id, errors))
# revert to initial commit hash
AgentGitHandler.execute_git_command(["reset", "--hard", init_head],
git_repo.local_repo_path)
return
# push to remote
try:
repo = Repo(git_repo.local_repo_path)
push_info = repo.remotes.origin.push()
if str(push_info[0].summary) is "[rejected] (fetch first)":
# need to pull
repo.remotes.origin.pull()
if repo.is_dirty():
# auto merge failed, need to reset
# TODO: what to do here?
raise GitRepositorySynchronizationException(
"Git pull before push operation left repository in dirty state."
)
# pull successful, now push
repo.remotes.origin.push()
AgentGitHandler.log.debug("Pushed artifacts for tenant : %s" %
git_repo.tenant_id)
except (GitCommandError, GitRepositorySynchronizationException) as e:
# revert to initial commit hash
AgentGitHandler.execute_git_command(["reset", "--hard", init_head],
git_repo.local_repo_path)
raise GitRepositorySynchronizationException(
"Pushing artifacts to remote repository failed for tenant %s: %s"
% (git_repo.tenant_id, e))
