def _clone_repo(repo_url): user_home = os.path.expanduser('~') # Assuming git url is of form [email protected]:user/git-repo.git repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')] abs_local_path = os.path.join(user_home, repo_name) Repo.clone_from(repo_url, abs_local_path) return abs_local_path
def _clone_repo(repo_url, branch='master'): user_home = os.path.expanduser('~') # Assuming git url is of form [email protected]:user/git-repo.git repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')] abs_local_path = os.path.join(user_home, repo_name) Repo.clone_from(repo_url, abs_local_path, branch=branch)) return abs_local_path
def _clone_repo(repo_url, verifyssl=True, branch='master'): user_home = os.path.expanduser('~') # Assuming git url is of form [email protected]:user/git-repo.git repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')] abs_local_path = os.path.join(user_home, repo_name) # Disable SSL cert checking if explictly asked if not verifyssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' Repo.clone_from(repo_url, abs_local_path, branch=branch) return abs_local_path
def _clone_repo(repo_url, verifyssl=True, branch="master"): user_home = os.path.expanduser("~") # Assuming git url is of form [email protected]:user/git-repo.git repo_name = DownloadGitRepoAction._eval_repo_name(repo_url) abs_local_path = os.path.join(user_home, repo_name) # Disable SSL cert checking if explictly asked if not verifyssl: os.environ["GIT_SSL_NO_VERIFY"] = "true" # Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a # specific branch so save some download time. Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1) return abs_local_path
def _clone_repo(repo_url, verifyssl=True, branch='master'): user_home = os.path.expanduser('~') # Assuming git url is of form [email protected]:user/git-repo.git repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')] abs_local_path = os.path.join(user_home, repo_name) # Disable SSL cert checking if explictly asked if not verifyssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' # Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a # specific branch so save some download time. Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1) return abs_local_path
def _clone_repo(repo_url, verifyssl=True, branch='master'): user_home = os.path.expanduser('~') # Assuming git url is of form [email protected]:user/git-repo.git repo_name = DownloadGitRepoAction._eval_repo_name(repo_url) abs_local_path = os.path.join(user_home, repo_name) # Disable SSL cert checking if explictly asked if not verifyssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' # Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a # specific branch so save some download time. Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1) return abs_local_path
def code_clone(code_link): download_link = 'https://github.com/' + code_link.split( '/')[3] + '/' + code_link.split('/')[4] + '.git' to_dir = './git_repository/' + code_link.split('/')[4] try: Repo.clone_from(download_link, to_dir) except: if os.path.exists('./git_repository/' + code_link.split('/')[4]): print('-----duplicated project-----') else: print('something wrong happen in git clone') else: print('-----successfully download: ' + download_link + '-----')
def install(): bar = window("install") tmpDir = make_tmp_dir("install") Logger.info( "HexOSBase: Started downloading the newest version of HexOS from github" ) Repo.clone_from(globals.baseSysConfig.get("os_changes", "src"), tmpDir) copy( bar, os.path.join(tmpDir, globals.baseSysConfig.get("main", "name") + "Files")) Logger.info( "HexOSBase: Finished downloading the newest version of HexOS from github" )
def git_pull(remote_url, local, idc_dir): """ @remote: git remote address @local: local git repo dir """ commit_msg = None if not os.path.exists(local): os.mkdir(local) logging.info('git start pull from remote repo.') if not os.path.exists( "{0}/.git".format(local)) or not git.repo.fun.is_git_dir( "{0}/.git".format(local)): repo_clone = Repo.clone_from(remote_url, local) if not repo_clone: logging.debug("clone repo from {0} to {1}failed.\n".format( remote_url, local)) return repo = Repo(local) remote = None git_commit = git.Git(idc_dir) last_commit_log = "/var/log/last_commit.log" last_commit = None if os.path.exists(last_commit_log): with file(last_commit_log) as fhand: last_log_commit = fhand.read().strip() # Warning: for performance only check 500 commit, use this write mysql for commit in repo.iter_commits('master', max_count=500): if last_commit == commit.hexsha: last_commit = last_log_commit break if not last_commit: last_commit = repo.head.commit.hexsha with FileLock(os.path.split(local)[1]) as _flock: for remote_name in repo.remotes: if remote_name.url == remote_url: remote = remote_name break if remote is None: remote_name = "%x" % (random.randint(0, 1000000) ^ int(time.time())) remote = repo.create_remote(remote_name, remote_url) ret = 0 try: info_list = remote.pull() for info in info_list: ret = info.flags & (git.FetchInfo.REJECTED | git.FetchInfo.ERROR) if ret > 0: logging.warning("[conf-pull] pull from remote error!\n") commit_msg = git_commit.log("{0}..HEAD".format(last_commit), "--pretty=%H,%ai,%s", idc_dir) except (git.GitCommandError, git.RepositoryDirtyError), e: logging.warning("git pull error, error={0}\n".format(str(e))) except Exception: logging.warning("git pull error, other exception.\n")
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref=None): user_home = os.path.expanduser('~') abs_local_path = os.path.join(user_home, temp_dir) # Switch to non-interactive mode os.environ['GIT_TERMINAL_PROMPT'] = '0' # Disable SSL cert checking if explictly asked if not verifyssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' if not ref: ref = 'master' # Clone the repo from git; we don't use shallow copying # because we want the user to work with the repo in the # future. repo = Repo.clone_from(repo_url, abs_local_path, branch='master') if not DownloadGitRepoAction._ref_exists(repo, ref): if re.match(SEMVER_REGEX, ref) and DownloadGitRepoAction._ref_exists(repo, "v%s" % ref): ref = "v%s" % ref else: raise ValueError("\"%s\" is not a valid ref in %s." % (ref, repo_url)) repo.head.reference = repo.commit(ref) repo.head.reset(index=True, working_tree=True) return abs_local_path
def setup(self): git_opts = self._config if git_opts['url'] is None: raise Exception('Remote git URL not set.') self._url = git_opts['url'] self._branch = git_opts['branch'] default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones') self._local_path = git_opts.get('local_clone_path', default_clone_dir) self._poll_interval = git_opts.get('poll_interval', self._poll_interval) if os.path.exists(self._local_path): self._repo = Repo.init(self._local_path) else: try: self._repo = Repo.clone_from(self._url, self._local_path, branch=self._branch) except Exception: self._logger.exception( 'Unable to clone remote repo from %s branch %s', self._url, self._branch) raise self._remote = self._repo.remote('origin')
def _prepare_local_repository(self, opts): url = opts['url'] branch = opts['branch'] if url is None: raise Exception('Remote git URL not set.') # set local repository path to be cloned repo_name = url[url.rindex('/') + 1:] default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones', repo_name) local_path = opts.get('local_clone_path', default_clone_dir) # create a directory to store cloned repositories if not os.path.exists(os.path.dirname(local_path)): os.mkdir(os.path.dirname(local_path), 0o755) if os.path.exists(local_path): repo_local = Repo.init(local_path) else: try: repo_local = Repo.clone_from(url, local_path, branch=branch) except Exception as e: raise Exception('Unable to clone remote repo from %s [%s]: %s' % (url, branch, str(e))) return repo_local, repo_local.remote('origin')
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'): # Switch to non-interactive mode os.environ['GIT_TERMINAL_PROMPT'] = '0' os.environ['GIT_ASKPASS'] = '******' # Disable SSL cert checking if explictly asked if not verifyssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' # Clone the repo from git; we don't use shallow copying # because we want the user to work with the repo in the # future. repo = Repo.clone_from(repo_url, temp_dir) use_branch = False # Try to match the reference to a branch name (i.e. "master") gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref) if gitref: use_branch = True # Try to match the reference to a commit hash, a tag, or "master" if not gitref: gitref = DownloadGitRepoAction._get_gitref(repo, ref) # Try to match the reference to a "vX.Y.Z" tag if not gitref and re.match(PACK_VERSION_REGEX, ref): gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref) # Giving up ¯\_(ツ)_/¯ if not gitref: format_values = [ref, repo_url] msg = '"%s" is not a valid version, hash, tag or branch in %s.' valid_versions = DownloadGitRepoAction._get_valid_versions_for_repo( repo=repo) if len(valid_versions) >= 1: valid_versions_string = ', '.join(valid_versions) msg += ' Available versions are: %s.' format_values.append(valid_versions_string) raise ValueError(msg % tuple(format_values)) # We're trying to figure out which branch the ref is actually on, # since there's no direct way to check for this in git-python. branches = repo.git.branch('-a', '--contains', gitref.hexsha) branches = branches.replace('*', '').split() if 'master' not in branches or use_branch: branch = "origin/%s" % ref if use_branch else branches[0] short_branch = ref if use_branch else branches[0].split('/')[-1] repo.git.checkout('-b', short_branch, branch) branch = repo.head.reference else: branch = 'master' repo.git.checkout(gitref.hexsha) repo.git.branch('-f', branch, gitref.hexsha) repo.git.checkout(branch) return temp_dir
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'): # Switch to non-interactive mode os.environ['GIT_TERMINAL_PROMPT'] = '0' os.environ['GIT_ASKPASS'] = '******' # Disable SSL cert checking if explictly asked if not verifyssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' # Clone the repo from git; we don't use shallow copying # because we want the user to work with the repo in the # future. repo = Repo.clone_from(repo_url, temp_dir) use_branch = False # Try to match the reference to a branch name (i.e. "master") gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref) if gitref: use_branch = True # Try to match the reference to a commit hash, a tag, or "master" if not gitref: gitref = DownloadGitRepoAction._get_gitref(repo, ref) # Try to match the reference to a "vX.Y.Z" tag if not gitref and re.match(PACK_VERSION_REGEX, ref): gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref) # Giving up ¯\_(ツ)_/¯ if not gitref: format_values = [ref, repo_url] msg = '"%s" is not a valid version, hash, tag or branch in %s.' valid_versions = DownloadGitRepoAction._get_valid_versions_for_repo(repo=repo) if len(valid_versions) >= 1: valid_versions_string = ', '.join(valid_versions) msg += ' Available versions are: %s.' format_values.append(valid_versions_string) raise ValueError(msg % tuple(format_values)) # We're trying to figure out which branch the ref is actually on, # since there's no direct way to check for this in git-python. branches = repo.git.branch('-a', '--contains', gitref.hexsha) branches = branches.replace('*', '').split() if 'master' not in branches or use_branch: branch = "origin/%s" % ref if use_branch else branches[0] short_branch = ref if use_branch else branches[0].split('/')[-1] repo.git.checkout('-b', short_branch, branch) branch = repo.head.reference else: branch = 'master' repo.git.checkout(gitref.hexsha) repo.git.branch('-f', branch, gitref.hexsha) repo.git.checkout(branch) return temp_dir
def cloneRepos(json_file): global app_cnt download_path = os.path.join(work_dir, "repos") with open(os.path.join(work_dir, json_file)) as json_fd: load_dict = json.load(json_fd) for idx, item in enumerate(load_dict["items"]): if app_cnt == 100: print(8 * '=', "100 application cloned", 8 * '=') return clone_url = item["clone_url"] name = item["full_name"] author = item["owner"]["login"] repo_name = author + "-" + item["name"] if os.path.exists( os.path.join(work_dir, json_file_pos + "not_app_list")): with open( os.path.join(work_dir, json_file_pos + "not_app_list"), 'r+') as not_app_list: lines = [line.strip() for line in not_app_list.readlines()] found = False for line in lines: if line == repo_name.strip(): found = True break if found: continue if os.path.exists(os.path.join(download_path, repo_name)) is False: Repo.clone_from(clone_url, to_path=os.path.join(download_path, repo_name)) if os.path.exists(os.path.join(download_path, repo_name, "app")): print("repo {} cloned".format(name)) app_cnt += 1 repos_coll.append(item) # with open(os.path.join(work_dir, json_file_pos + "repo_list"), # 'w') as repo_list: # repo_list.write(repo_name + "\n") else: with open( os.path.join(work_dir, json_file_pos + "not_app_list"), 'a') as not_app_list: not_app_list.write(repo_name + "\n") rmtree(os.path.join(download_path, repo_name)) cmd = "rm -rf " + os.path.join(download_path, repo_name) os.system(cmd) print("repo {} added to blacklist".format(repo_name))
def git_pull(remote_url, git_local): """ @remote_url: git remote address @git_local: local git repo dir """ logging.info('git start pull from remote repo.') # check if we need to clean the local git clean_local_git(False, git_local, remote_url) # git clone if not exist if not os.path.exists("{0}/.git".format(git_local)) or \ not git.repo.fun.is_git_dir("{0}/.git".format(git_local)): logging.info('git clone from [%s] to [%s], and depth=1.', remote_url, git_local) repo_clone = Repo.clone_from(remote_url, git_local,depth=1) if not repo_clone: logging.error( "clone repo from {0} to {1}failed.".format(remote_url, git_local)) logging.info('========run stop========\n\n') sys.exit(1) # get repo repo = Repo(git_local) type(repo.git).GIT_PYTHON_TRACE = 'full' remote = None # get remote url for item in repo.remotes: if item.url == remote_url: remote = item break if remote is None: clean_local_git(True, git_local, remote_url) # start pull ssh_executable = os.path.join(g_script_path, 'debug_ssh.sh') with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_executable): try: logging.info('git pull from remote=%s, url=%s', remote.name, remote.url) info_list = remote.pull() for info in info_list: ret = info.flags & (git.FetchInfo.REJECTED | git.FetchInfo.ERROR) if ret > 0: logging.warning("git pull from remote ERROR: %s", info_list) logging.info('========run stop========\n\n') sys.exit(1) else: logging.info("git pull return mesg: %s", info_list) except BadName: logging.warn("git pull warn: ", exc_info=True) except Exception: clean_local_git(True, git_local, remote_url) # get last commit msg last_commit_id = str(repo.head.commit) return last_commit_id
def on_get(self, req, res): repoPath = './IntallPlist' #本地仓库路径 result = 'success' #默认返回success msg = [] repo = None # 判断本地是否有仓库 if os.path.exists(repoPath): repo = Repo(repoPath) msg.append('已有仓库') else: repo = Repo.clone_from( url='[email protected]:hillyoung/TestPlist.git', to_path=repoPath) msg.append('不存在仓库') git = repo.git git.checkout('*') remote = repo.remote() remote.pull() msg.append('完成拉取远程仓库代码') # 判断是否传入了文件名 name = req.params.get('name', None) if name is None: msg.append('传入无效参数') result = 'fail' else: msg.append('传入应用名:' + name) # 构造plist文件内容及相关控制逻辑 directoryPath = repoPath + '/' + name if os.path.exists(directoryPath) != True: os.mkdir(directoryPath) filePath = directoryPath + '/install.plist' #plist文件相对路径 rContent = None #用于缓存仓库中plist文件的内容 flag = 'w' #文件的打开访问,默认以写入方式打开 if os.path.exists(filePath): #判断仓库中是否存在plist文件 flag = 'r+' #如果plist文件存在,则以读写的方式打开 msg.append('存在plist文件') content = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"><plist version="1.0"><dict><key>items</key><array><dict><key>assets</key><array><dict><key>kind</key><string>software-package</string><key>url</key><string>https://raw.githubusercontent.com/hillyoung/AppInstaller/master/app/' + name + '/LiemsMobileEnterprise.ipa</string></dict></array><key>metadata</key><dict><key>bundle-identifier</key> <string>net.luculent.liems.hhgs</string><key>bundle-version</key><string>1.0.1</string><key>kind</key><string>software</string><key>title</key><string>测试安装</string></dict></dict></array></dict></plist>' with open(filePath, flag) as flie_object: #写入plist if flag == 'r+': rContent = flie_object.read() flie_object.seek(0) msg.append(rContent) if rContent != content: #写入plist内容 flie_object.write(content) flie_object.close() #提交plist文件 git.add('*') git.commit('-m', '添加' + name) remote.push() res.body = json.dumps({'result': result, 'msg': msg}) res.status = falcon.HTTP_200 # This is the default status res.append_header('Access-Control-Allow-Origin', '*')
def initial(self, branch): if not os.path.exists(self.local_path): os.makedirs(self.local_path) git_local_path = os.path.join(self.local_path, '.git') if not is_git_dir(git_local_path): self.repo = Repo.clone_from(self.repo_url, to_path=self.local_path, branch=branch) else: self.repo = Repo(self.local_path)
def __init__(self, database): logging.info("Initializing GitSync.") self.database = database self.charts = dict() self.url = DEFAULT_GITREPO try: self.repo = Repo(REPO_DIRECTORY) except InvalidGitRepositoryError: logging.info("Cloning repository in %s", REPO_DIRECTORY) self.repo = Repo.clone_from(self.url, REPO_DIRECTORY)
def _setup_repo(self, already_cloned): if already_cloned: self._git_repo = Repo(self.repo_dir_path) else: _log.info('Cloning repo from %s into %s' % ( self._github_repo_url, self.repo_dir_path)) self._git_repo = Repo.clone_from( self._github_repo_url, self.repo_dir_path) _log.info('-Done-') if self.branch: self._git_repo.git.checkout(self.branch)
def clone_git_repo(git_url): ''' Name : clone_git_repo Input : git_url - url for the git repository Ouptput : None Description: This function takes git url and local path as input and clone the git repo to that local dir ''' #BASE CASE #check for empty input if not len(git_url): print('Empty git repo url.Please enter a valid git url') return #clone the repo try: print('Git clone is in progress...') #local code repo path repo_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),'git_code_repo') if not os.path.isdir(repo_dir): os.mkdir(repo_dir) #clone the repo Repo.clone_from(git_url, repo_dir) print('{giturl} is cloned succesfully to {repodir}'.format(giturl=git_url,repodir=repo_dir)) except Exception as e: print('Git clone failed with error - {error} - for giturl:{giturl}'.format(error=e,giturl=git_url))
def initial(self, repo_url, branch): """ 初始化git仓库 :param repo_url: :param branch: :return: """ if not os.path.exists(self.local_path): os.makedirs(self.local_path) git_local_path = os.path.join(self.local_path, '.git') if not is_git_dir(git_local_path): self.repo = Repo.clone_from(repo_url, to_path=self.local_path, branch=branch) else: self.repo = Repo(self.local_path)
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'): # Switch to non-interactive mode os.environ['GIT_TERMINAL_PROMPT'] = '0' # Disable SSL cert checking if explictly asked if not verifyssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' # Clone the repo from git; we don't use shallow copying # because we want the user to work with the repo in the # future. repo = Repo.clone_from(repo_url, temp_dir) # Try to match the reference to a commit hash, a tag, or "master" gitref = DownloadGitRepoAction._get_gitref(repo, ref) # Try to match the reference to a "vX.Y.Z" tag if not gitref and re.match(SEMVER_REGEX, ref): gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref) # Try to match the reference to a branch name if not gitref: gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref) # Giving up ¯\_(ツ)_/¯ if not gitref: raise ValueError( "\"%s\" is not a valid version, hash, tag, or branch in %s." % (ref, repo_url)) # We're trying to figure out which branch the ref is actually on, # since there's no direct way to check for this in git-python. branches = repo.git.branch('--color=never', '--all', '--contains', gitref.hexsha) branches = branches.replace('*', '').split() if 'master' not in branches: branch = branches[0] repo.git.checkout('--track', branches[0]) branch = repo.head.reference else: branch = 'master' repo.git.checkout('-B', branch, gitref.hexsha) return temp_dir
def initial(self, repo_url, branch): """ 初始化git仓库 :param repo_url: :param branch: :return: """ if os.path.exists(self.local_path): shutil.rmtree(self.local_path, ignore_errors=True) os.makedirs(self.local_path) print("clone empty-repo from remote:{0} to local:{1}".format( repo_url, self.local_path)) print("please wait...\n") self.repo = Repo.clone_from(repo_url, to_path=self.local_path, branch=branch, bare=True)
def _clone(self, local_dir: t.Optional[str] = None, config_username: t.Optional[str] = None, config_user_email: t.Optional[str] = None): """Clone the specified repository and recursively clone submodules. This method raises a git.exec.GitCommandError if the repository does not exist. """ if local_dir: self.local_dir = local_dir else: self.local_dir = tempfile.mkdtemp() self.repo = Repo.clone_from(self.url, self.local_dir, multi_options=['--recurse-submodules']) if config_username: with self.repo.config_writer() as config: config.set_value('user', 'email', config_user_email) config.set_value('user', 'name', config_username)
def get_repo(repo_url: str, repo_path: Path): if repo_path.exists(): repo = Repo(repo_path) heads = Git().ls_remote(repo_url) git_hash_length = 40 remote_last_commit = heads.split("\n")[0][:git_hash_length] if remote_last_commit != repo.head.commit.hexsha: update_repo = questionary.confirm( "The remote repository has been updated." "Do you want to update it?" ).ask() if update_repo: repo.remotes.origin.pull() else: # pyright: reportUnknownMemberType=false repo = Repo.clone_from(repo_url, repo_path) return repo
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'): # Switch to non-interactive mode os.environ['GIT_TERMINAL_PROMPT'] = '0' # Disable SSL cert checking if explictly asked if not verifyssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' # Clone the repo from git; we don't use shallow copying # because we want the user to work with the repo in the # future. repo = Repo.clone_from(repo_url, temp_dir) # Try to match the reference to a commit hash, a tag, or "master" gitref = DownloadGitRepoAction._get_gitref(repo, ref) # Try to match the reference to a "vX.Y.Z" tag if not gitref and re.match(SEMVER_REGEX, ref): gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref) # Try to match the reference to a branch name if not gitref: gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref) # Giving up ¯\_(ツ)_/¯ if not gitref: raise ValueError( "\"%s\" is not a valid version, hash, tag, or branch in %s." % (ref, repo_url) ) # We're trying to figure out which branch the ref is actually on, # since there's no direct way to check for this in git-python. branches = repo.git.branch('--color=never', '--all', '--contains', gitref.hexsha) branches = branches.replace('*', '').split() if 'master' not in branches: branch = branches[0] repo.git.checkout('--track', branches[0]) branch = repo.head.reference else: branch = 'master' repo.git.checkout('-B', branch, gitref.hexsha) return temp_dir
def _set_up_repo_state(local_base, git_remote, org, git_local, repo_name, branch, commit, post_clone): git_base = jpth(git_remote, org, repo_name) if not os.path.exists(local_base): repo = Repo.clone_from(git_base + '.git', local_base) post_clone() # FIXME if this does not complete we need to warn or something, it causes errors else: repo = Repo(local_base) nob = repo.active_branch try: nab = getBranch(repo, branch) nab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) nab = repo.active_branch repo.remote().pull() # make sure we are up to date if commit != 'HEAD': repo.git.checkout(commit) return repo, nob
def setup(self): git_opts = self._config if git_opts['url'] is None: raise Exception('Remote git URL not set.') self._url = git_opts['url'] default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones') self._local_path = git_opts.get('local_clone_path', default_clone_dir) self._poll_interval = git_opts.get('poll_interval', self._poll_interval) if os.path.exists(self._local_path): self._repo = Repo.init(self._local_path) else: try: self._repo = Repo.clone_from(self._url, self._local_path) except Exception: self._logger.exception('Unable to clone remote repo from %s', self._url) raise self._remote = self._repo.remote('origin')
def main(): # identify local packages in the environment and clone if they don't exist for package in environment.local_packages: pkg_path = locate_local_package(package) prj_repo_name = f"{DEFAULT_BRANCH}-{PROJECT_NAME}" # if package is local, switch to base branch "dev-init" if pkg_path: # switch branch to dev_init repo = Repo(pkg_path) if repo.active_branch.name != BASE_BRANCH: repo.git.checkout(BASE_BRANCH) repo.git.pull() # otherwise clone the package from github else: print(f'Cloning {package}') repo = Repo.clone_from(url=f'{RP_GITHUB_URL}/{package}', to_path=str(pkg_path), branch=BASE_BRANCH) # check if the project branch exists, if not create it if not check_branch(pkg_path, prj_repo_name): repo.create_head(prj_repo_name) # check out the project branch and push up to github repo.git.checkout(prj_repo_name) repo.git.push()
def clean_local_git(clean_now, git_local, remote_url): #判断目录是否存在 if not os.path.exists(git_local): logging.info('git_local[%s] not exist, no need to clean.', git_local) return #判断是否为清理时间(凌晨4点-24点) if not time_in_range(time(4, 0), time(23, 0)) and not clean_now: logging.info('git_local[%s] now[%s], not in clean time range[4:00-23:00].', git_local, datetime.now().strftime('%Y-%m-%d %H:%M:%S')) return #判断今天是否已经清理过 if is_already_cleaned(git_local, clean_now): logging.info('git_local[%s] is already cleaned, just continue.', git_local) return #清理 git_local_new = git_local + '.new' if os.path.exists(git_local_new): logging.info('rm dir[%s].', git_local_new) rm_dir(git_local_new) logging.info('git clone from [%s] to [%s], and depth=1.', remote_url, git_local_new) repo_clone = Repo.clone_from(remote_url, git_local_new, depth=1) if not repo_clone: logging.error( "clone repo from {0} to {1}failed.".format(remote_url, git_local_new)) logging.info('========run stop========\n\n') sys.exit(1) git_local_bak = git_local + '.bak' if os.path.exists(git_local_bak): logging.info('rm dir[%s].', git_local_bak) rm_dir(git_local_bak) logging.info('mv dir[%s] to dir[%s].', git_local, git_local_bak) shutil.move(git_local, git_local_bak) logging.info('mv dir[%s] to dir[%s].', git_local_new, git_local) shutil.move(git_local_new, git_local)
import os from git.repo import Repo giturl = 'https://github.com/Andy-Jen/openCV_ws' file_path = 'gitserver' local_path = os.path.join(file_path) if os.path.exists(file_path): print('Find git server, check version...') repo = Repo(local_path) for tag in repo.tags: print(tag.name) else: print('Nofind git server, Git clone for giturl...') Repo.clone_from(giturl, to_path=local_path, branch='master')
def clone_repo(self, name, path): Repo.clone_from("{}@{}:{}/{}.git".format(self.user,self.name,self.root,name),path)
def scigraph_build(zip_location, git_remote, org, git_local, branch, commit, clean=False, check_built=False, cleanup_later=False, quiet=False): COMMIT_LOG = 'last-built-commit.log' repo_name = 'SciGraph' remote = jpth(git_remote, org, repo_name) local = jpth(git_local, repo_name) commit_log_path = jpth(local, COMMIT_LOG) if not os.path.exists(local): repo = Repo.clone_from(remote + '.git', local) elif not Path(local, '.git').exists(): repo = Repo.clone_from(remote + '.git', local) else: repo = Repo(local) if not os.path.exists(commit_log_path): last_commit = None else: with open(commit_log_path, 'rt') as f: last_commit = f.read().strip() sob = repo.active_branch try: sab = getBranch(repo, branch) sab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) sab = repo.active_branch repo.remote().pull() if commit != 'HEAD': repo.git.checkout(commit) scigraph_commit = repo.head.object.hexsha scigraph_commit_short = scigraph_commit[:COMMIT_HASH_HEAD_LEN] bin_location = zip_location / 'bin' os.environ['PATH'] = bin_location.as_posix() + ':' + os.environ.get( 'PATH', '') if not bin_location.exists(): bin_location.mkdir() # hack to make the scigraph-load we are about to create available as a command # so that it matches the usual scigraph-load behavior def zip_name(wild=False): return (repo_name + '-' + branch + '-services' + '-' + ('*' if wild else TODAY()) + '-' + scigraph_commit_short + '.zip') def reset_state(original_branch=sob): original_branch.checkout() with execute_regardless( reset_state, only_exception=cleanup_later ): # FIXME this fails when we need to load the graph if we start on master :/ # main if scigraph_commit != last_commit or clean: print('SciGraph not built at commit', commit, 'last built at', last_commit) quiet = '--quiet ' if quiet else '' build_command = ( 'cd ' + local + f';export HASH={scigraph_commit_short}' ';sed -i "/<name>SciGraph<\/name>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" pom.xml' ';sed -i "/<artifactId>scigraph<\/artifactId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-analysis/pom.xml' ';sed -i "/<groupId>io.scigraph<\/groupId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-core/pom.xml' ';sed -i "/<artifactId>scigraph<\/artifactId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-entity/pom.xml' ';sed -i "/<groupId>io.scigraph<\/groupId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-services/pom.xml' f'; mvn {quiet}clean -DskipTests -DskipITs install' '; cd SciGraph-services' f'; mvn {quiet}-DskipTests -DskipITs package') if check_built: print('SciGraph has not been built.') raise NotBuiltError('SciGraph has not been built.') out = os.system(build_command) print(out) if out: scigraph_commit = 'FAILURE' with open(commit_log_path, 'wt') as f: f.write(scigraph_commit) else: print('SciGraph already built at commit', scigraph_commit) wildcard = jpth(zip_location, zip_name(wild=True)) try: services_zip = glob(wildcard)[ 0] # this will error if the zip was moved return scigraph_commit, services_zip, reset_state except IndexError: pass # we need to copy the zip out again # services zip zip_filename = f'scigraph-services-{scigraph_commit_short}.zip' services_zip_temp = Path(local, 'SciGraph-services', 'target', zip_filename) services_zip = jpth(zip_location, zip_name()) shutil.copy(services_zip_temp, services_zip) core_jar = Path( local, 'SciGraph-core', 'target', f'scigraph-core-{scigraph_commit_short}-jar-with-dependencies.jar') scigraph_load = f'''#!/usr/bin/env sh /usr/bin/java \\ -cp "{core_jar.as_posix()}" \\ io.scigraph.owlapi.loader.BatchOwlLoader $@''' slf = bin_location / 'scigraph-load' with open(slf, 'wt') as f: f.write(scigraph_load) os.chmod(slf, 0o0755) return scigraph_commit, services_zip, reset_state
def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'): # Switch to non-interactive mode os.environ['GIT_TERMINAL_PROMPT'] = '0' os.environ['GIT_ASKPASS'] = '******' # Disable SSL cert checking if explictly asked if not verify_ssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' # Clone the repo from git; we don't use shallow copying # because we want the user to work with the repo in the # future. repo = Repo.clone_from(repo_url, temp_dir) is_local_repo = repo_url.startswith('file://') try: active_branch = repo.active_branch except TypeError as e: if is_local_repo: active_branch = None else: raise e # Special case for local git repos - we allow users to install from repos which are checked out # at a specific commit (aka detached HEAD) if is_local_repo and not active_branch and not ref: LOG.debug( 'Installing pack from git repo on disk, skipping branch checkout') return temp_dir use_branch = False # Special case when a default repo branch is not "master" # No ref provided so we just use a default active branch if (not ref or ref == active_branch.name ) and repo.active_branch.object == repo.head.commit: gitref = repo.active_branch.object else: # Try to match the reference to a branch name (i.e. "master") gitref = get_gitref(repo, 'origin/%s' % ref) if gitref: use_branch = True # Try to match the reference to a commit hash, a tag, or "master" if not gitref: gitref = get_gitref(repo, ref) # Try to match the reference to a "vX.Y.Z" tag if not gitref and re.match(PACK_VERSION_REGEX, ref): gitref = get_gitref(repo, 'v%s' % ref) # Giving up ¯\_(ツ)_/¯ if not gitref: format_values = [ref, repo_url] msg = '"%s" is not a valid version, hash, tag or branch in %s.' valid_versions = get_valid_versions_for_repo(repo=repo) if len(valid_versions) >= 1: valid_versions_string = ', '.join(valid_versions) msg += ' Available versions are: %s.' format_values.append(valid_versions_string) raise ValueError(msg % tuple(format_values)) # We're trying to figure out which branch the ref is actually on, # since there's no direct way to check for this in git-python. branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member branches = branches.replace('*', '').split() if active_branch.name not in branches or use_branch: branch = 'origin/%s' % ref if use_branch else branches[0] short_branch = ref if use_branch else branches[0].split('/')[-1] repo.git.checkout('-b', short_branch, branch) branch = repo.head.reference else: branch = repo.active_branch.name repo.git.checkout(gitref.hexsha) # pylint: disable=no-member repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member repo.git.checkout(branch) return temp_dir
def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'): # Switch to non-interactive mode os.environ['GIT_TERMINAL_PROMPT'] = '0' os.environ['GIT_ASKPASS'] = '******' # Disable SSL cert checking if explictly asked if not verify_ssl: os.environ['GIT_SSL_NO_VERIFY'] = 'true' # Clone the repo from git; we don't use shallow copying # because we want the user to work with the repo in the # future. repo = Repo.clone_from(repo_url, temp_dir) is_local_repo = repo_url.startswith('file://') try: active_branch = repo.active_branch except TypeError as e: if is_local_repo: active_branch = None else: raise e # Special case for local git repos - we allow users to install from repos which are checked out # at a specific commit (aka detached HEAD) if is_local_repo and not active_branch and not ref: LOG.debug('Installing pack from git repo on disk, skipping branch checkout') return temp_dir use_branch = False # Special case when a default repo branch is not "master" # No ref provided so we just use a default active branch if (not ref or ref == active_branch.name) and repo.active_branch.object == repo.head.commit: gitref = repo.active_branch.object else: # Try to match the reference to a branch name (i.e. "master") gitref = get_gitref(repo, 'origin/%s' % ref) if gitref: use_branch = True # Try to match the reference to a commit hash, a tag, or "master" if not gitref: gitref = get_gitref(repo, ref) # Try to match the reference to a "vX.Y.Z" tag if not gitref and re.match(PACK_VERSION_REGEX, ref): gitref = get_gitref(repo, 'v%s' % ref) # Giving up ¯\_(ツ)_/¯ if not gitref: format_values = [ref, repo_url] msg = '"%s" is not a valid version, hash, tag or branch in %s.' valid_versions = get_valid_versions_for_repo(repo=repo) if len(valid_versions) >= 1: valid_versions_string = ', '.join(valid_versions) msg += ' Available versions are: %s.' format_values.append(valid_versions_string) raise ValueError(msg % tuple(format_values)) # We're trying to figure out which branch the ref is actually on, # since there's no direct way to check for this in git-python. branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member branches = branches.replace('*', '').split() if active_branch.name not in branches or use_branch: branch = 'origin/%s' % ref if use_branch else branches[0] short_branch = ref if use_branch else branches[0].split('/')[-1] repo.git.checkout('-b', short_branch, branch) branch = repo.head.reference else: branch = repo.active_branch.name repo.git.checkout(gitref.hexsha) # pylint: disable=no-member repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member repo.git.checkout(branch) return temp_dir
def scigraph_build(zip_location, git_remote, org, git_local, branch, commit, clean=False, check_built=False, cleanup_later=False): COMMIT_LOG = 'last-built-commit.log' repo_name = 'SciGraph' remote = jpth(git_remote, org, repo_name) local = jpth(git_local, repo_name) commit_log_path = jpth(local, COMMIT_LOG) load_base = ('cd {}; '.format(jpth(local, 'SciGraph-core')) + 'mvn exec:java ' '-Dexec.mainClass="io.scigraph.owlapi.loader.BatchOwlLoader" ' '-Dexec.args="-c {config_path}"') if not os.path.exists(local): repo = Repo.clone_from(remote + '.git', local) else: repo = Repo(local) if not os.path.exists(commit_log_path): last_commit = None else: with open(commit_log_path, 'rt') as f: last_commit = f.read().strip() sob = repo.active_branch try: sab = getBranch(repo, branch) sab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) sab = repo.active_branch repo.remote().pull() if commit != 'HEAD': repo.git.checkout(commit) scigraph_commit = repo.head.object.hexsha def zip_name(wild=False): return (repo_name + '-' + branch + '-services' + '-' + ('*' if wild else TODAY()) + '-' + scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '.zip') def reset_state(original_branch=sob): original_branch.checkout() with execute_regardless( reset_state, only_exception=cleanup_later ): # FIXME this fails when we need to load the graph if we start on master :/ # main if scigraph_commit != last_commit or clean: print('SciGraph not built at commit', commit, 'last built at', last_commit) build_command = ('cd ' + local + '; mvn clean -DskipTests -DskipITs install' '; cd SciGraph-services' '; mvn -DskipTests -DskipITs package') if check_built: print('SciGraph has not been built.') raise NotBuiltError('SciGraph has not been built.') out = os.system(build_command) print(out) if out: scigraph_commit = 'FAILURE' with open(commit_log_path, 'wt') as f: f.write(scigraph_commit) else: print('SciGraph already built at commit', scigraph_commit) wildcard = jpth(zip_location, zip_name(wild=True)) try: services_zip = glob(wildcard)[ 0] # this will error if the zip was moved return scigraph_commit, load_base, services_zip, reset_state except IndexError: pass # we need to copy the zip out again # services zip zip_filename = 'scigraph-services-*-SNAPSHOT.zip' services_zip_temp = glob( jpth(local, 'SciGraph-services', 'target', zip_filename))[0] services_zip = jpth(zip_location, zip_name()) shutil.copy(services_zip_temp, services_zip) return scigraph_commit, load_base, services_zip, reset_state
def __init__(self, zip_location, git_remote, org, git_local, repo_name, branch, commit, remote_base, load_base, graphload_config, patch_config, patch, scigraph_commit, post_clone=lambda: None, check_built=False): local_base = jpth(git_local, repo_name) git_base = jpth(git_remote, org, repo_name) if not os.path.exists(local_base): repo = Repo.clone_from(git_base + '.git', local_base) post_clone( ) # FIXME if this does not complete we need to warn or something, it causes errors else: repo = Repo(local_base) nob = repo.active_branch try: nab = getBranch(repo, branch) nab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) nab = repo.active_branch repo.remote().pull() # make sure we are up to date if commit != 'HEAD': repo.git.checkout(commit) # TODO consider dumping metadata in a file in the folder too? def folder_name(scigraph_commit, wild=False): ontology_commit = repo.head.object.hexsha[:COMMIT_HASH_HEAD_LEN] return (repo_name + '-' + branch + '-graph' + '-' + ('*' if wild else TODAY()) + '-' + scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '-' + ontology_commit) def make_folder_zip(wild=False): folder = folder_name(scigraph_commit, wild) graph_path = jpth(zip_location, folder) zip_path = graph_path + '.zip' if wild: return graph_path, zip_path zip_name = os.path.basename(zip_path) zip_dir = os.path.dirname(zip_path) zip_command = ' '.join( ('cd', zip_dir, ';', 'zip -r', zip_name, folder)) return graph_path, zip_path, zip_command graph_path, zip_path, zip_command = make_folder_zip() wild_graph_path, wild_zip_path = make_folder_zip(wild=True) (config, config_path, ontologies) = self.make_graphload_config(graphload_config, graph_path, remote_base, local_base, zip_location) load_command = load_base.format( config_path=config_path) # 'exit 1' to test print(load_command) # replace raw github imports with ontology.neuinfor iris to simplify import chain fix_imports = "find " + local_base + " -name '*.ttl' -exec sed -i 's/<http.\+\/ttl\//<http:\/\/ontology.neuinfo.org\/NIF\/ttl\//' {} \;" os.system(fix_imports) def reset_state(original_branch=nob): repo.git.checkout('--', local_base) original_branch.checkout() with execute_regardless( reset_state ): # FIXME start this immediately after we obtain nob? # main if patch: # FIXME TODO XXX does scigraph load from the catalog!??!?? # because it seems like doid loads correctly without using local_versions # which would be cool, if confusing local_versions = tuple(do_patch(patch_config, local_base)) else: local_versions = tuple() itrips = local_imports( remote_base, local_base, ontologies, local_versions=local_versions, dobig=True) # SciGraph doesn't support catalog.xml catalog = make_catalog(itrips) with open(Path(local_base, 'catalog.xml'), 'wt') as f: f.write(catalog) maybe_zip_path = glob(wild_zip_path) if not maybe_zip_path: if check_built: print('The graph has not been loaded.') raise NotBuiltError('The graph has not been loaded.') failure = os.system(load_command) if failure: if os.path.exists(graph_path): shutil.rmtree(graph_path) else: os.rename( config_path, # save the config for eaiser debugging jpth(graph_path, os.path.basename(config_path))) failure = os.system(zip_command) # graphload zip else: zip_path = maybe_zip_path[0] # this way we get the actual date print('Graph already loaded at', zip_path) # this needs to be run when the branch is checked out # FIXME might be worth adding this to the load config? self.ontologies = [ get_iri(load_header(rec['url'])) for rec in config['ontologies'] ] self.zip_path = zip_path self.itrips = itrips self.config = config