def add_remote(git_repo): # add origin remote output, errors = AgentGitHandler.execute_git_command( ["remote", "add", "origin", git_repo.repo_url], git_repo.local_repo_path) if len(output) > 0: raise GitRepositorySynchronizationException( "Error in adding remote origin %s for local repository %s" % (git_repo.repo_url, git_repo.local_repo_path)) # fetch output, errors = AgentGitHandler.execute_git_command( ["fetch"], git_repo.local_repo_path) if "Resolving deltas: 100%" not in output: raise GitRepositorySynchronizationException( "Error in fetching from remote origin %s for local repository %s" % (git_repo.repo_url, git_repo.local_repo_path)) # checkout master output, errors = AgentGitHandler.execute_git_command( ["checkout", "master"], git_repo.local_repo_path) if "Branch master set up to track remote branch master from origin." not in output: raise GitRepositorySynchronizationException( "Error in checking out master branch %s for local repository %s" % (git_repo.repo_url, git_repo.local_repo_path)) return True
def pull(git_repo): # git reset to make sure no uncommitted changes are present before the pull, no conflicts will occur AgentGitHandler.execute_git_command(["reset", "--hard"], git_repo.local_repo_path) # HEAD before pull (init_head, init_errors) = AgentGitHandler.execute_git_command( ["rev-parse", "HEAD"], git_repo.local_repo_path) try: repo = Repo(git_repo.local_repo_path) repo.remotes.origin.pull() if repo.is_dirty(): raise GitRepositorySynchronizationException( "Git pull operation left the repository in dirty state") except (GitCommandError, GitRepositorySynchronizationException) as e: raise GitRepositorySynchronizationException( "Git pull operation on %s for tenant %s failed: %s" % (git_repo.repo_url, git_repo.tenant_id, e)) # HEAD after pull (end_head, end_errors) = AgentGitHandler.execute_git_command( ["rev-parse", "HEAD"], git_repo.local_repo_path) # check if HEAD was updated if init_head != end_head: AgentGitHandler.log.debug( "Artifacts were updated as a result of the pull operation, thread: %s - %s" % (current_thread().getName(), current_thread().ident)) return True else: return False
def retry_clone(git_repo): """Retry 'git clone' operation for defined number of attempts with defined intervals """ git_clone_successful = False # Read properties from agent.conf max_retry_attempts = int(Config.artifact_clone_retry_count) retry_interval = int(Config.artifact_clone_retry_interval) retry_attempts = 0 # Iterate until git clone is successful or reaches max retry attempts while git_clone_successful is False and retry_attempts < max_retry_attempts: try: retry_attempts += 1 AgentGitHandler.clone(git_repo) AgentGitHandler.log.info( "Retrying attempt to git clone operation for tenant %s successful" % git_repo.tenant_id) git_clone_successful = True except GitRepositorySynchronizationException as e: AgentGitHandler.log.exception( "Retrying git clone attempt %s failed: %s" % (retry_attempts, e)) if retry_attempts < max_retry_attempts: time.sleep(retry_interval) else: raise GitRepositorySynchronizationException( "All attempts failed while retrying git clone: %s" % e)
def clone(git_repo): try: # create a temporary location to clone temp_repo_path = os.path.join(tempfile.gettempdir(), "pca_temp_" + git_repo.tenant_id) if os.path.isdir( temp_repo_path) and os.listdir(temp_repo_path) != []: GitUtils.delete_folder_tree(temp_repo_path) GitUtils.create_dir(temp_repo_path) # clone the repo to a temporary location first to avoid conflicts AgentGitHandler.log.debug( "Cloning artifacts from URL: %s to temp location: %s" % (git_repo.repo_url, temp_repo_path)) Repo.clone_from(git_repo.auth_url, temp_repo_path) # clear the paths to get rid of the following bug in distutils: # http://stackoverflow.com/questions/9160227/dir-util-copy-tree-fails-after-shutil-rmtree distutils.dir_util._path_created = {} # move the cloned dir to application path copy_tree(temp_repo_path, git_repo.local_repo_path) AgentGitHandler.log.info( "Git clone operation for tenant %s successful" % git_repo.tenant_id) return git_repo except GitCommandError as e: raise GitRepositorySynchronizationException( "Error while cloning repository for tenant %s: %s" % (git_repo.tenant_id, e))
def clone(git_repo): try: # create a temporary location to clone temp_repo_path = os.path.join(tempfile.gettempdir(), "pca_temp_" + git_repo.tenant_id) if os.path.isdir( temp_repo_path) and os.listdir(temp_repo_path) != []: GitUtils.delete_folder_tree(temp_repo_path) GitUtils.create_dir(temp_repo_path) # clone the repo to a temporary location first to avoid conflicts AgentGitHandler.log.debug( "Cloning artifacts from URL: %s to temp location: %s" % (git_repo.repo_url, temp_repo_path)) Repo.clone_from(git_repo.auth_url, temp_repo_path, branch=git_repo.branch) # move the cloned dir to application path copy_tree(temp_repo_path, git_repo.local_repo_path) AgentGitHandler.log.info( "Git clone operation for tenant %s successful" % git_repo.tenant_id) return git_repo except GitCommandError as e: raise GitRepositorySynchronizationException( "Error while cloning repository for tenant %s: %s" % (git_repo.tenant_id, e))
def delete_folder_tree(path): """ Completely deletes the provided folder :param str path: Full path of the folder :return: void """ try: shutil.rmtree(path) GitUtils.log.debug("Directory [%s] deleted." % path) except OSError as e: raise GitRepositorySynchronizationException("Deletion of folder path %s failed: %s" % (path, e))
def create_dir(path): """ mkdir the provided path :param path: The path to the directory to be made :return: True if mkdir was successful, False if dir already exists :rtype: bool """ try: os.mkdir(path) GitUtils.log.debug("Successfully created directory [%s]" % path) # return True except OSError as e: raise GitRepositorySynchronizationException("Directory creating failed in [%s]. " % e)
def clone(git_repo): if os.path.isdir(git_repo.local_repo_path): # delete and recreate local repo path if exists AgentGitHandler.log.debug( "Local repository path not empty. Cleaning.") GitUtils.delete_folder_tree(git_repo.local_repo_path) try: Repo.clone_from(git_repo.repo_url, git_repo.local_repo_path) AgentGitHandler.add_repo(git_repo) AgentGitHandler.log.info( "Git clone operation for tenant %s successful" % git_repo.tenant_id) return git_repo except GitCommandError as e: raise GitRepositorySynchronizationException( "Error while cloning repository: %s" % e)
def retry_clone(git_repo): """Retry 'git clone' operation for defined number of attempts with defined intervals """ if os.path.isdir(git_repo.local_repo_path) and os.listdir( git_repo.local_repo_path) != []: # delete and recreate local repo path if not empty dir AgentGitHandler.log.debug( "Local repository path not empty. Cleaning.") GitUtils.delete_folder_tree(git_repo.local_repo_path) GitUtils.create_dir(git_repo.local_repo_path) git_clone_successful = False # Read properties from agent.conf max_retry_attempts = int( Config.read_property(constants.ARTIFACT_CLONE_RETRIES, 5)) retry_interval = int( Config.read_property(constants.ARTIFACT_CLONE_INTERVAL, 10)) retry_attempts = 0 # Iterate until git clone is successful or reaches max retry attempts while git_clone_successful is False and retry_attempts < max_retry_attempts: try: retry_attempts += 1 Repo.clone_from(git_repo.repo_url, git_repo.local_repo_path) AgentGitHandler.add_repo(git_repo) AgentGitHandler.log.info( "Retrying attempt to git clone operation for tenant %s successful" % git_repo.tenant_id) git_clone_successful = True except GitCommandError as e: AgentGitHandler.log.warn( "Retrying git clone attempt %s failed" % retry_attempts) if retry_attempts < max_retry_attempts: time.sleep(retry_interval) pass else: raise GitRepositorySynchronizationException( "Error while retrying git clone: %s" % e)
def push(repo_info): """ Commits and pushes new artifacts to the remote repository :param repo_info: :return: """ git_repo = AgentGitHandler.get_repo(repo_info.tenant_id) if git_repo is None: # not cloned yet raise GitRepositorySynchronizationException( "Not a valid repository to push from. Aborting") # Get initial HEAD so in case if push fails it can be reverted to this hash # This way, commit and push becomes an single operation. No intermediate state will be left behind. (init_head, init_errors) = AgentGitHandler.execute_git_command( ["rev-parse", "HEAD"], git_repo.local_repo_path) # stage all untracked files if AgentGitHandler.stage_all(git_repo.local_repo_path): AgentGitHandler.log.info( "Git staged untracked artifacts successfully") else: AgentGitHandler.log.info("Git could not stage untracked artifacts") # check if modified files are present modified = AgentGitHandler.has_modified_files(git_repo.local_repo_path) AgentGitHandler.log.debug("[Git] Modified: %s" % str(modified)) if not modified: AgentGitHandler.log.debug( "No changes detected in the local repository for tenant %s" % git_repo.tenant_id) return # commit to local repositpory commit_message = "tenant [%s]'s artifacts committed to local repo at %s" \ % (git_repo.tenant_id, git_repo.local_repo_path) # TODO: set configuratble names, check if already configured commit_name = git_repo.tenant_id commit_email = "*****@*****.**" # git config AgentGitHandler.execute_git_command( ["config", "user.email", commit_email], git_repo.local_repo_path) AgentGitHandler.execute_git_command( ["config", "user.name", commit_name], git_repo.local_repo_path) # commit (output, errors) = AgentGitHandler.execute_git_command( ["commit", "-m", commit_message], git_repo.local_repo_path) if errors.strip() == "": commit_hash = AgentGitHandler.find_between(output, "[master", "]").strip() AgentGitHandler.log.debug( "Committed artifacts for tenant : %s : %s " % (git_repo.tenant_id, commit_hash)) else: AgentGitHandler.log.exception( "Committing artifacts to local repository failed for tenant: %s, Cause: %s" % (git_repo.tenant_id, errors)) # revert to initial commit hash AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) return # push to remote try: repo = Repo(git_repo.local_repo_path) push_info = repo.remotes.origin.push() if str(push_info[0].summary) is "[rejected] (fetch first)": # need to pull repo.remotes.origin.pull() if repo.is_dirty(): # auto merge failed, need to reset # TODO: what to do here? raise GitRepositorySynchronizationException( "Git pull before push operation left repository in dirty state." ) # pull successful, now push repo.remotes.origin.push() AgentGitHandler.log.debug("Pushed artifacts for tenant : %s" % git_repo.tenant_id) except (GitCommandError, GitRepositorySynchronizationException) as e: # revert to initial commit hash AgentGitHandler.execute_git_command(["reset", "--hard", init_head], git_repo.local_repo_path) raise GitRepositorySynchronizationException( "Pushing artifacts to remote repository failed for tenant %s: %s" % (git_repo.tenant_id, e))