def checkout(self, repo_info):
"""
Checks out the code from the remote repository.
If local repository path is empty, a clone operation is done.
If there is a cloned repository already on the local repository path, a pull operation
will be performed.
If there are artifacts not in the repository already on the local repository path,
they will be added to a git repository, the remote url added as origin, and then
a pull operation will be performed.
:param Repository repo_info: The repository information object
:return: A tuple containing whether it was an initial clone or not, and if the repo was updated on
subsequent calls or not
:rtype: tuple(bool, bool)
"""
new_git_repo = AgentGitHandler.create_git_repo(repo_info)
# check whether this is the first artifact updated event for this tenant
existing_git_repo = AgentGitHandler.get_repo(repo_info.tenant_id)
if existing_git_repo is not None:
# check whether this event has updated credentials for git repo
if AgentGitHandler.is_valid_git_repository(
new_git_repo
) and new_git_repo.repo_url != existing_git_repo.repo_url:
# add the new git_repo object with updated credentials to repo list
AgentGitHandler.add_repo(new_git_repo)
# update the origin remote URL with new credentials
self.log.info(
"Changes detected in git credentials for tenant: %s" %
new_git_repo.tenant_id)
(output, errors) = AgentGitHandler.execute_git_command(
["remote", "set-url", "origin", new_git_repo.repo_url],
new_git_repo.local_repo_path)
if errors.strip() != "":
self.log.error(
"Failed to update git repo remote URL for tenant: %s" %
new_git_repo.tenant_id)
git_repo = AgentGitHandler.create_git_repo(repo_info)
if AgentGitHandler.get_repo(repo_info.tenant_id) is not None:
# has been previously cloned, this is not the subscription run
if AgentGitHandler.is_valid_git_repository(git_repo):
self.log.debug(
"Executing git pull: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
updated = AgentGitHandler.pull(git_repo)
self.log.debug(
"Git pull executed: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
else:
# not a valid repository, might've been corrupted. do a re-clone
self.log.debug(
"Local repository is not valid. Doing a re-clone to purify."
)
git_repo.cloned = False
self.log.debug(
"Executing git clone: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
git_repo = AgentGitHandler.clone(git_repo)
AgentGitHandler.add_repo(git_repo)
self.log.debug(
"Git clone executed: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
else:
# subscribing run.. need to clone
self.log.info("Cloning artifacts from %s for the first time to %s",
git_repo.repo_url, git_repo.local_repo_path)
self.log.info(
"Executing git clone: [tenant-id] %s [repo-url] %s, [repo path] %s",
git_repo.tenant_id, git_repo.repo_url,
git_repo.local_repo_path)
try:
git_repo = AgentGitHandler.clone(git_repo)
AgentGitHandler.add_repo(git_repo)
self.log.debug(
"Git clone executed: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
except Exception as e:
self.log.exception("Git clone operation failed: %s" % e)
# If first git clone is failed, execute retry_clone operation
self.log.info("Retrying git clone operation...")
AgentGitHandler.retry_clone(git_repo)
AgentGitHandler.add_repo(git_repo)
def commit(self, repo_info):
"""
Commits and pushes new artifacts to the remote repository
:param repo_info:
:return:
"""
git_repo = AgentGitHandler.get_repo(repo_info.tenant_id)
if git_repo is None:
# not cloned yet
AgentGitHandler.log.error("Not a valid repository to push from. Aborting Git push...")
# Get initial HEAD so in case if push fails it can be reverted to this hash
# This way, commit and push becomes an single operation. No intermediate state will be left behind.
(init_head, init_errors) = AgentGitHandler.execute_git_command(["rev-parse", "HEAD"], git_repo.local_repo_path)
# remove trailing new line character, if any
init_head = init_head.rstrip()
# stage all untracked files
if AgentGitHandler.stage_all(git_repo.local_repo_path):
AgentGitHandler.log.debug("Git staged untracked artifacts successfully")
else:
AgentGitHandler.log.error("Git could not stage untracked artifacts")
# check for changes in working directory
modified = AgentGitHandler.has_modified_files(git_repo.local_repo_path)
AgentGitHandler.log.debug("[Git] Modified: %s" % str(modified))
if not modified:
AgentGitHandler.log.debug("No changes detected in the local repository for tenant %s" % git_repo.tenant_id)
return
# commit to local repository
commit_message = "tenant [%s]'s artifacts committed to local repo at %s" \
% (git_repo.tenant_id, git_repo.local_repo_path)
# TODO: set configuratble names, check if already configured
commit_name = git_repo.tenant_id
commit_email = "*****@*****.**"
# git config
AgentGitHandler.execute_git_command(["config", "user.email", commit_email], git_repo.local_repo_path)
AgentGitHandler.execute_git_command(["config", "user.name", commit_name], git_repo.local_repo_path)
# commit
(output, errors) = AgentGitHandler.execute_git_command(["commit", "-m", commit_message],
git_repo.local_repo_path)
if errors.strip() == "":
commit_hash = AgentGitHandler.find_between(output, "[master", "]").strip()
AgentGitHandler.log.debug("Committed artifacts for tenant: %s : %s " % (git_repo.tenant_id, commit_hash))
else:
AgentGitHandler.log.error("Committing artifacts to local repository failed for tenant: %s, Cause: %s"
% (git_repo.tenant_id, errors))
# revert to initial commit hash
AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path)
return
repo = Repo(git_repo.local_repo_path)
# pull and rebase before pushing to remote repo
AgentGitHandler.execute_git_command(["pull", "--rebase", "origin", "master"], git_repo.local_repo_path)
if repo.is_dirty():
AgentGitHandler.log.error("Git pull operation in commit job left the repository in dirty state")
AgentGitHandler.log.error(
"Git pull rebase operation on remote %s for tenant %s failed" % (git_repo.repo_url, git_repo.tenant_id))
AgentGitHandler.log.warn("The working directory will be reset to the last known good commit")
# revert to the initial commit
AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path)
return
else:
# push to remote
try:
push_info_list = repo.remotes.origin.push()
if (len(push_info_list)) == 0:
AgentGitHandler.log.error("Failed to push artifacts to remote repo for tenant: %s remote: %s" %
(git_repo.tenant_id, git_repo.repo_url))
# revert to the initial commit
AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path)
return
for push_info in push_info_list:
AgentGitHandler.log.debug("Push info summary: %s" % push_info.summary)
if push_info.flags & PushInfo.ERROR == PushInfo.ERROR:
AgentGitHandler.log.error("Failed to push artifacts to remote repo for tenant: %s remote: %s" %
(git_repo.tenant_id, git_repo.repo_url))
# revert to the initial commit
AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path)
return
AgentGitHandler.log.debug(
"Successfully pushed artifacts for tenant: %s remote: %s" % (git_repo.tenant_id, git_repo.repo_url))
except Exception as e:
AgentGitHandler.log.error(
"Failed to push artifacts to remote repo for tenant: %s remote: %s exception: %s" %
(git_repo.tenant_id, git_repo.repo_url, e))
# revert to the initial commit
AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path)
def checkout(self, repo_info):
"""
Checks out the code from the remote repository.
If local repository path is empty, a clone operation is done.
If there is a cloned repository already on the local repository path, a pull operation
will be performed.
If there are artifacts not in the repository already on the local repository path,
they will be added to a git repository, the remote url added as origin, and then
a pull operation will be performed.
:param Repository repo_info: The repository information object
:return: A tuple containing whether it was an initial clone or not, and if the repo was updated on
subsequent calls or not
:rtype: tuple(bool, bool)
"""
new_git_repo = AgentGitHandler.create_git_repo(repo_info)
# check whether this is the first artifact updated event for this tenant
existing_git_repo = AgentGitHandler.get_repo(repo_info.tenant_id)
if existing_git_repo is not None:
# check whether this event has updated credentials for git repo
if AgentGitHandler.is_valid_git_repository(
new_git_repo) and new_git_repo.repo_url != existing_git_repo.repo_url:
# add the new git_repo object with updated credentials to repo list
AgentGitHandler.add_repo(new_git_repo)
# update the origin remote URL with new credentials
self.log.info("Changes detected in git credentials for tenant: %s" % new_git_repo.tenant_id)
(output, errors) = AgentGitHandler.execute_git_command(
["remote", "set-url", "origin", new_git_repo.repo_url], new_git_repo.local_repo_path)
if errors.strip() != "":
self.log.error("Failed to update git repo remote URL for tenant: %s" % new_git_repo.tenant_id)
git_repo = AgentGitHandler.create_git_repo(repo_info)
if AgentGitHandler.get_repo(repo_info.tenant_id) is not None:
# has been previously cloned, this is not the subscription run
if AgentGitHandler.is_valid_git_repository(git_repo):
self.log.debug("Executing git pull: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
updated = AgentGitHandler.pull(git_repo)
self.log.debug("Git pull executed: [tenant-id] %s [repo-url] %s [SUCCESS] %s",
git_repo.tenant_id, git_repo.repo_url, updated)
else:
# not a valid repository, might've been corrupted. do a re-clone
self.log.debug("Local repository is not valid. Doing a re-clone to purify.")
git_repo.cloned = False
self.log.debug("Executing git clone: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
git_repo = AgentGitHandler.clone(git_repo)
AgentGitHandler.add_repo(git_repo)
self.log.debug("Git clone executed: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
else:
# subscribing run.. need to clone
self.log.info("Cloning artifacts from %s for the first time to %s",
git_repo.repo_url, git_repo.local_repo_path)
self.log.info("Executing git clone: [tenant-id] %s [repo-url] %s, [repo path] %s",
git_repo.tenant_id, git_repo.repo_url, git_repo.local_repo_path)
if Config.backup_initial_artifacts:
self.check_and_backup_initial_artifacts(git_repo.local_repo_path)
else:
self.log.info("Default artifact backup disabled")
try:
git_repo = AgentGitHandler.clone(git_repo)
AgentGitHandler.add_repo(git_repo)
self.log.debug("Git clone executed: [tenant-id] %s [repo-url] %s",
git_repo.tenant_id, git_repo.repo_url)
except Exception as e:
self.log.exception("Git clone operation failed: %s" % e)
# If first git clone is failed, execute retry_clone operation
self.log.info("Retrying git clone operation...")
AgentGitHandler.retry_clone(git_repo)
AgentGitHandler.add_repo(git_repo)
def commit(self, repo_info):
"""
Commits and pushes new artifacts to the remote repository
:param repo_info:
:return:
"""
git_repo = AgentGitHandler.get_repo(repo_info.tenant_id)
if git_repo is None:
# not cloned yet
AgentGitHandler.log.error(
"Not a valid repository to push from. Aborting Git push...")
# Get initial HEAD so in case if push fails it can be reverted to this hash
# This way, commit and push becomes an single operation. No intermediate state will be left behind.
(init_head, init_errors) = AgentGitHandler.execute_git_command(
["rev-parse", "HEAD"], git_repo.local_repo_path)
# remove trailing new line character, if any
init_head = init_head.rstrip()
# stage all untracked files
if AgentGitHandler.stage_all(git_repo.local_repo_path):
AgentGitHandler.log.debug(
"Git staged untracked artifacts successfully")
else:
AgentGitHandler.log.error(
"Git could not stage untracked artifacts")
# check for changes in working directory
modified = AgentGitHandler.has_modified_files(git_repo.local_repo_path)
AgentGitHandler.log.debug("[Git] Modified: %s" % str(modified))
if not modified:
AgentGitHandler.log.debug(
"No changes detected in the local repository for tenant %s" %
git_repo.tenant_id)
return
# commit to local repository
commit_message = "tenant [%s]'s artifacts committed to local repo at %s" \
% (git_repo.tenant_id, git_repo.local_repo_path)
# TODO: set configuratble names, check if already configured
commit_name = git_repo.tenant_id
commit_email = "*****@*****.**"
# git config
AgentGitHandler.execute_git_command(
["config", "user.email", commit_email], git_repo.local_repo_path)
AgentGitHandler.execute_git_command(
["config", "user.name", commit_name], git_repo.local_repo_path)
# commit
(output, errors) = AgentGitHandler.execute_git_command(
["commit", "-m", commit_message], git_repo.local_repo_path)
if errors.strip() == "":
commit_hash = AgentGitHandler.find_between(output, "[master",
"]").strip()
AgentGitHandler.log.debug(
"Committed artifacts for tenant: %s : %s " %
(git_repo.tenant_id, commit_hash))
else:
AgentGitHandler.log.error(
"Committing artifacts to local repository failed for tenant: %s, Cause: %s"
% (git_repo.tenant_id, errors))
# revert to initial commit hash
AgentGitHandler.execute_git_command(["reset", "--hard", init_head],
git_repo.local_repo_path)
return
repo = Repo(git_repo.local_repo_path)
# pull and rebase before pushing to remote repo
AgentGitHandler.execute_git_command(
["pull", "--rebase", "origin", "master"], git_repo.local_repo_path)
if repo.is_dirty():
AgentGitHandler.log.error(
"Git pull operation in commit job left the repository in dirty state"
)
AgentGitHandler.log.error(
"Git pull rebase operation on remote %s for tenant %s failed" %
(git_repo.repo_url, git_repo.tenant_id))
AgentGitHandler.log.warn(
"The working directory will be reset to the last known good commit"
)
# revert to the initial commit
AgentGitHandler.execute_git_command(["reset", "--hard", init_head],
git_repo.local_repo_path)
return
else:
# push to remote
try:
push_info_list = repo.remotes.origin.push()
if (len(push_info_list)) == 0:
AgentGitHandler.log.error(
"Failed to push artifacts to remote repo for tenant: %s remote: %s"
% (git_repo.tenant_id, git_repo.repo_url))
# revert to the initial commit
AgentGitHandler.execute_git_command(
["reset", "--hard", init_head],
git_repo.local_repo_path)
return
for push_info in push_info_list:
AgentGitHandler.log.debug("Push info summary: %s" %
push_info.summary)
if push_info.flags & PushInfo.ERROR == PushInfo.ERROR:
AgentGitHandler.log.error(
"Failed to push artifacts to remote repo for tenant: %s remote: %s"
% (git_repo.tenant_id, git_repo.repo_url))
# revert to the initial commit
AgentGitHandler.execute_git_command(
["reset", "--hard", init_head],
git_repo.local_repo_path)
return
AgentGitHandler.log.debug(
"Successfully pushed artifacts for tenant: %s remote: %s" %
(git_repo.tenant_id, git_repo.repo_url))
except Exception as e:
AgentGitHandler.log.error(
"Failed to push artifacts to remote repo for tenant: %s remote: %s exception: %s"
% (git_repo.tenant_id, git_repo.repo_url, e))
# revert to the initial commit
AgentGitHandler.execute_git_command(
["reset", "--hard", init_head], git_repo.local_repo_path)
