def _clone_repo(repo_url):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')]
abs_local_path = os.path.join(user_home, repo_name)
Repo.clone_from(repo_url, abs_local_path)
return abs_local_path
def _clone_repo(repo_url, branch='master'):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')]
abs_local_path = os.path.join(user_home, repo_name)
Repo.clone_from(repo_url, abs_local_path, branch=branch))
return abs_local_path
def _clone_repo(repo_url, verifyssl=True, branch='master'):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')]
abs_local_path = os.path.join(user_home, repo_name)
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
Repo.clone_from(repo_url, abs_local_path, branch=branch)
return abs_local_path
def _clone_repo(repo_url, verifyssl=True, branch="master"):
user_home = os.path.expanduser("~")
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = DownloadGitRepoAction._eval_repo_name(repo_url)
abs_local_path = os.path.join(user_home, repo_name)
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ["GIT_SSL_NO_VERIFY"] = "true"
# Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a
# specific branch so save some download time.
Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1)
return abs_local_path
def _clone_repo(repo_url, verifyssl=True, branch='master'):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = repo_url[repo_url.rfind('/') + 1: repo_url.rfind('.')]
abs_local_path = os.path.join(user_home, repo_name)
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a
# specific branch so save some download time.
Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1)
return abs_local_path
def _clone_repo(repo_url, verifyssl=True, branch='master'):
user_home = os.path.expanduser('~')
# Assuming git url is of form [email protected]:user/git-repo.git
repo_name = DownloadGitRepoAction._eval_repo_name(repo_url)
abs_local_path = os.path.join(user_home, repo_name)
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Shallow clone the repo to avoid getting all the metadata. We only need HEAD of a
# specific branch so save some download time.
Repo.clone_from(repo_url, abs_local_path, branch=branch, depth=1)
return abs_local_path
def code_clone(code_link):
download_link = 'https://github.com/' + code_link.split(
'/')[3] + '/' + code_link.split('/')[4] + '.git'
to_dir = './git_repository/' + code_link.split('/')[4]
try:
Repo.clone_from(download_link, to_dir)
except:
if os.path.exists('./git_repository/' + code_link.split('/')[4]):
print('-----duplicated project-----')
else:
print('something wrong happen in git clone')
else:
print('-----successfully download: ' + download_link + '-----')
def install():
bar = window("install")
tmpDir = make_tmp_dir("install")
Logger.info(
"HexOSBase: Started downloading the newest version of HexOS from github"
)
Repo.clone_from(globals.baseSysConfig.get("os_changes", "src"), tmpDir)
copy(
bar,
os.path.join(tmpDir,
globals.baseSysConfig.get("main", "name") + "Files"))
Logger.info(
"HexOSBase: Finished downloading the newest version of HexOS from github"
)
def git_pull(remote_url, local, idc_dir):
"""
@remote: git remote address
@local: local git repo dir
"""
commit_msg = None
if not os.path.exists(local):
os.mkdir(local)
logging.info('git start pull from remote repo.')
if not os.path.exists(
"{0}/.git".format(local)) or not git.repo.fun.is_git_dir(
"{0}/.git".format(local)):
repo_clone = Repo.clone_from(remote_url, local)
if not repo_clone:
logging.debug("clone repo from {0} to {1}failed.\n".format(
remote_url, local))
return
repo = Repo(local)
remote = None
git_commit = git.Git(idc_dir)
last_commit_log = "/var/log/last_commit.log"
last_commit = None
if os.path.exists(last_commit_log):
with file(last_commit_log) as fhand:
last_log_commit = fhand.read().strip()
# Warning: for performance only check 500 commit, use this write mysql
for commit in repo.iter_commits('master', max_count=500):
if last_commit == commit.hexsha:
last_commit = last_log_commit
break
if not last_commit:
last_commit = repo.head.commit.hexsha
with FileLock(os.path.split(local)[1]) as _flock:
for remote_name in repo.remotes:
if remote_name.url == remote_url:
remote = remote_name
break
if remote is None:
remote_name = "%x" % (random.randint(0, 1000000)
^ int(time.time()))
remote = repo.create_remote(remote_name, remote_url)
ret = 0
try:
info_list = remote.pull()
for info in info_list:
ret = info.flags & (git.FetchInfo.REJECTED
| git.FetchInfo.ERROR)
if ret > 0:
logging.warning("[conf-pull] pull from remote error!\n")
commit_msg = git_commit.log("{0}..HEAD".format(last_commit),
"--pretty=%H,%ai,%s", idc_dir)
except (git.GitCommandError, git.RepositoryDirtyError), e:
logging.warning("git pull error, error={0}\n".format(str(e)))
except Exception:
logging.warning("git pull error, other exception.\n")
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref=None):
user_home = os.path.expanduser('~')
abs_local_path = os.path.join(user_home, temp_dir)
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
if not ref:
ref = 'master'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, abs_local_path, branch='master')
if not DownloadGitRepoAction._ref_exists(repo, ref):
if re.match(SEMVER_REGEX, ref) and DownloadGitRepoAction._ref_exists(repo, "v%s" % ref):
ref = "v%s" % ref
else:
raise ValueError("\"%s\" is not a valid ref in %s." % (ref, repo_url))
repo.head.reference = repo.commit(ref)
repo.head.reset(index=True, working_tree=True)
return abs_local_path
def setup(self):
git_opts = self._config
if git_opts['url'] is None:
raise Exception('Remote git URL not set.')
self._url = git_opts['url']
self._branch = git_opts['branch']
default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones')
self._local_path = git_opts.get('local_clone_path', default_clone_dir)
self._poll_interval = git_opts.get('poll_interval',
self._poll_interval)
if os.path.exists(self._local_path):
self._repo = Repo.init(self._local_path)
else:
try:
self._repo = Repo.clone_from(self._url,
self._local_path,
branch=self._branch)
except Exception:
self._logger.exception(
'Unable to clone remote repo from %s branch %s', self._url,
self._branch)
raise
self._remote = self._repo.remote('origin')
def _prepare_local_repository(self, opts):
url = opts['url']
branch = opts['branch']
if url is None:
raise Exception('Remote git URL not set.')
# set local repository path to be cloned
repo_name = url[url.rindex('/') + 1:]
default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones', repo_name)
local_path = opts.get('local_clone_path', default_clone_dir)
# create a directory to store cloned repositories
if not os.path.exists(os.path.dirname(local_path)):
os.mkdir(os.path.dirname(local_path), 0o755)
if os.path.exists(local_path):
repo_local = Repo.init(local_path)
else:
try:
repo_local = Repo.clone_from(url, local_path, branch=branch)
except Exception as e:
raise Exception('Unable to clone remote repo from %s [%s]: %s' %
(url, branch, str(e)))
return repo_local, repo_local.remote('origin')
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
os.environ['GIT_ASKPASS'] = '******'
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, temp_dir)
use_branch = False
# Try to match the reference to a branch name (i.e. "master")
gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref)
if gitref:
use_branch = True
# Try to match the reference to a commit hash, a tag, or "master"
if not gitref:
gitref = DownloadGitRepoAction._get_gitref(repo, ref)
# Try to match the reference to a "vX.Y.Z" tag
if not gitref and re.match(PACK_VERSION_REGEX, ref):
gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref)
# Giving up ¯\_(ツ)_/¯
if not gitref:
format_values = [ref, repo_url]
msg = '"%s" is not a valid version, hash, tag or branch in %s.'
valid_versions = DownloadGitRepoAction._get_valid_versions_for_repo(
repo=repo)
if len(valid_versions) >= 1:
valid_versions_string = ', '.join(valid_versions)
msg += ' Available versions are: %s.'
format_values.append(valid_versions_string)
raise ValueError(msg % tuple(format_values))
# We're trying to figure out which branch the ref is actually on,
# since there's no direct way to check for this in git-python.
branches = repo.git.branch('-a', '--contains', gitref.hexsha)
branches = branches.replace('*', '').split()
if 'master' not in branches or use_branch:
branch = "origin/%s" % ref if use_branch else branches[0]
short_branch = ref if use_branch else branches[0].split('/')[-1]
repo.git.checkout('-b', short_branch, branch)
branch = repo.head.reference
else:
branch = 'master'
repo.git.checkout(gitref.hexsha)
repo.git.branch('-f', branch, gitref.hexsha)
repo.git.checkout(branch)
return temp_dir
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
os.environ['GIT_ASKPASS'] = '******'
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, temp_dir)
use_branch = False
# Try to match the reference to a branch name (i.e. "master")
gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref)
if gitref:
use_branch = True
# Try to match the reference to a commit hash, a tag, or "master"
if not gitref:
gitref = DownloadGitRepoAction._get_gitref(repo, ref)
# Try to match the reference to a "vX.Y.Z" tag
if not gitref and re.match(PACK_VERSION_REGEX, ref):
gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref)
# Giving up ¯\_(ツ)_/¯
if not gitref:
format_values = [ref, repo_url]
msg = '"%s" is not a valid version, hash, tag or branch in %s.'
valid_versions = DownloadGitRepoAction._get_valid_versions_for_repo(repo=repo)
if len(valid_versions) >= 1:
valid_versions_string = ', '.join(valid_versions)
msg += ' Available versions are: %s.'
format_values.append(valid_versions_string)
raise ValueError(msg % tuple(format_values))
# We're trying to figure out which branch the ref is actually on,
# since there's no direct way to check for this in git-python.
branches = repo.git.branch('-a', '--contains', gitref.hexsha)
branches = branches.replace('*', '').split()
if 'master' not in branches or use_branch:
branch = "origin/%s" % ref if use_branch else branches[0]
short_branch = ref if use_branch else branches[0].split('/')[-1]
repo.git.checkout('-b', short_branch, branch)
branch = repo.head.reference
else:
branch = 'master'
repo.git.checkout(gitref.hexsha)
repo.git.branch('-f', branch, gitref.hexsha)
repo.git.checkout(branch)
return temp_dir
def cloneRepos(json_file):
global app_cnt
download_path = os.path.join(work_dir, "repos")
with open(os.path.join(work_dir, json_file)) as json_fd:
load_dict = json.load(json_fd)
for idx, item in enumerate(load_dict["items"]):
if app_cnt == 100:
print(8 * '=', "100 application cloned", 8 * '=')
return
clone_url = item["clone_url"]
name = item["full_name"]
author = item["owner"]["login"]
repo_name = author + "-" + item["name"]
if os.path.exists(
os.path.join(work_dir, json_file_pos + "not_app_list")):
with open(
os.path.join(work_dir, json_file_pos + "not_app_list"),
'r+') as not_app_list:
lines = [line.strip() for line in not_app_list.readlines()]
found = False
for line in lines:
if line == repo_name.strip():
found = True
break
if found:
continue
if os.path.exists(os.path.join(download_path, repo_name)) is False:
Repo.clone_from(clone_url,
to_path=os.path.join(download_path, repo_name))
if os.path.exists(os.path.join(download_path, repo_name, "app")):
print("repo {} cloned".format(name))
app_cnt += 1
repos_coll.append(item)
# with open(os.path.join(work_dir, json_file_pos + "repo_list"),
# 'w') as repo_list:
# repo_list.write(repo_name + "\n")
else:
with open(
os.path.join(work_dir, json_file_pos + "not_app_list"),
'a') as not_app_list:
not_app_list.write(repo_name + "\n")
rmtree(os.path.join(download_path, repo_name))
cmd = "rm -rf " + os.path.join(download_path, repo_name)
os.system(cmd)
print("repo {} added to blacklist".format(repo_name))
def git_pull(remote_url, git_local):
"""
@remote_url: git remote address
@git_local: local git repo dir
"""
logging.info('git start pull from remote repo.')
# check if we need to clean the local git
clean_local_git(False, git_local, remote_url)
# git clone if not exist
if not os.path.exists("{0}/.git".format(git_local)) or \
not git.repo.fun.is_git_dir("{0}/.git".format(git_local)):
logging.info('git clone from [%s] to [%s], and depth=1.', remote_url, git_local)
repo_clone = Repo.clone_from(remote_url, git_local,depth=1)
if not repo_clone:
logging.error(
"clone repo from {0} to {1}failed.".format(remote_url, git_local))
logging.info('========run stop========\n\n')
sys.exit(1)
# get repo
repo = Repo(git_local)
type(repo.git).GIT_PYTHON_TRACE = 'full'
remote = None
# get remote url
for item in repo.remotes:
if item.url == remote_url:
remote = item
break
if remote is None:
clean_local_git(True, git_local, remote_url)
# start pull
ssh_executable = os.path.join(g_script_path, 'debug_ssh.sh')
with repo.git.custom_environment(GIT_SSH_COMMAND=ssh_executable):
try:
logging.info('git pull from remote=%s, url=%s',
remote.name, remote.url)
info_list = remote.pull()
for info in info_list:
ret = info.flags & (git.FetchInfo.REJECTED |
git.FetchInfo.ERROR)
if ret > 0:
logging.warning("git pull from remote ERROR: %s", info_list)
logging.info('========run stop========\n\n')
sys.exit(1)
else:
logging.info("git pull return mesg: %s", info_list)
except BadName:
logging.warn("git pull warn: ", exc_info=True)
except Exception:
clean_local_git(True, git_local, remote_url)
# get last commit msg
last_commit_id = str(repo.head.commit)
return last_commit_id
def on_get(self, req, res):
repoPath = './IntallPlist' #本地仓库路径
result = 'success' #默认返回success
msg = []
repo = None
# 判断本地是否有仓库
if os.path.exists(repoPath):
repo = Repo(repoPath)
msg.append('已有仓库')
else:
repo = Repo.clone_from(
url='[email protected]:hillyoung/TestPlist.git', to_path=repoPath)
msg.append('不存在仓库')
git = repo.git
git.checkout('*')
remote = repo.remote()
remote.pull()
msg.append('完成拉取远程仓库代码')
# 判断是否传入了文件名
name = req.params.get('name', None)
if name is None:
msg.append('传入无效参数')
result = 'fail'
else:
msg.append('传入应用名：' + name)
# 构造plist文件内容及相关控制逻辑
directoryPath = repoPath + '/' + name
if os.path.exists(directoryPath) != True:
os.mkdir(directoryPath)
filePath = directoryPath + '/install.plist' #plist文件相对路径
rContent = None #用于缓存仓库中plist文件的内容
flag = 'w' #文件的打开访问，默认以写入方式打开
if os.path.exists(filePath): #判断仓库中是否存在plist文件
flag = 'r+' #如果plist文件存在，则以读写的方式打开
msg.append('存在plist文件')
content = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"><plist version="1.0"><dict><key>items</key><array><dict><key>assets</key><array><dict><key>kind</key><string>software-package</string><key>url</key><string>https://raw.githubusercontent.com/hillyoung/AppInstaller/master/app/' + name + '/LiemsMobileEnterprise.ipa</string></dict></array><key>metadata</key><dict><key>bundle-identifier</key> <string>net.luculent.liems.hhgs</string><key>bundle-version</key><string>1.0.1</string><key>kind</key><string>software</string><key>title</key><string>测试安装</string></dict></dict></array></dict></plist>'
with open(filePath, flag) as flie_object: #写入plist
if flag == 'r+':
rContent = flie_object.read()
flie_object.seek(0)
msg.append(rContent)
if rContent != content: #写入plist内容
flie_object.write(content)
flie_object.close()
#提交plist文件
git.add('*')
git.commit('-m', '添加' + name)
remote.push()
res.body = json.dumps({'result': result, 'msg': msg})
res.status = falcon.HTTP_200 # This is the default status
res.append_header('Access-Control-Allow-Origin', '*')
def initial(self, branch):
if not os.path.exists(self.local_path):
os.makedirs(self.local_path)
git_local_path = os.path.join(self.local_path, '.git')
if not is_git_dir(git_local_path):
self.repo = Repo.clone_from(self.repo_url,
to_path=self.local_path,
branch=branch)
else:
self.repo = Repo(self.local_path)
def __init__(self, database):
logging.info("Initializing GitSync.")
self.database = database
self.charts = dict()
self.url = DEFAULT_GITREPO
try:
self.repo = Repo(REPO_DIRECTORY)
except InvalidGitRepositoryError:
logging.info("Cloning repository in %s", REPO_DIRECTORY)
self.repo = Repo.clone_from(self.url, REPO_DIRECTORY)
def _setup_repo(self, already_cloned):
if already_cloned:
self._git_repo = Repo(self.repo_dir_path)
else:
_log.info('Cloning repo from %s into %s' % (
self._github_repo_url, self.repo_dir_path))
self._git_repo = Repo.clone_from(
self._github_repo_url, self.repo_dir_path)
_log.info('-Done-')
if self.branch:
self._git_repo.git.checkout(self.branch)
def __init__(self, database):
logging.info("Initializing GitSync.")
self.database = database
self.charts = dict()
self.url = DEFAULT_GITREPO
try:
self.repo = Repo(REPO_DIRECTORY)
except InvalidGitRepositoryError:
logging.info("Cloning repository in %s", REPO_DIRECTORY)
self.repo = Repo.clone_from(self.url, REPO_DIRECTORY)
def clone_git_repo(git_url):
'''
Name : clone_git_repo
Input : git_url - url for the git repository
Ouptput : None
Description: This function takes git url and local path as input and clone the git repo to that local dir
'''
#BASE CASE
#check for empty input
if not len(git_url):
print('Empty git repo url.Please enter a valid git url')
return
#clone the repo
try:
print('Git clone is in progress...')
#local code repo path
repo_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),'git_code_repo')
if not os.path.isdir(repo_dir):
os.mkdir(repo_dir)
#clone the repo
Repo.clone_from(git_url, repo_dir)
print('{giturl} is cloned succesfully to {repodir}'.format(giturl=git_url,repodir=repo_dir))
except Exception as e:
print('Git clone failed with error - {error} - for giturl:{giturl}'.format(error=e,giturl=git_url))
def initial(self, repo_url, branch):
"""
初始化git仓库
:param repo_url:
:param branch:
:return:
"""
if not os.path.exists(self.local_path):
os.makedirs(self.local_path)
git_local_path = os.path.join(self.local_path, '.git')
if not is_git_dir(git_local_path):
self.repo = Repo.clone_from(repo_url, to_path=self.local_path, branch=branch)
else:
self.repo = Repo(self.local_path)
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, temp_dir)
# Try to match the reference to a commit hash, a tag, or "master"
gitref = DownloadGitRepoAction._get_gitref(repo, ref)
# Try to match the reference to a "vX.Y.Z" tag
if not gitref and re.match(SEMVER_REGEX, ref):
gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref)
# Try to match the reference to a branch name
if not gitref:
gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref)
# Giving up ¯\_(ツ)_/¯
if not gitref:
raise ValueError(
"\"%s\" is not a valid version, hash, tag, or branch in %s." %
(ref, repo_url))
# We're trying to figure out which branch the ref is actually on,
# since there's no direct way to check for this in git-python.
branches = repo.git.branch('--color=never', '--all', '--contains',
gitref.hexsha)
branches = branches.replace('*', '').split()
if 'master' not in branches:
branch = branches[0]
repo.git.checkout('--track', branches[0])
branch = repo.head.reference
else:
branch = 'master'
repo.git.checkout('-B', branch, gitref.hexsha)
return temp_dir
def initial(self, repo_url, branch):
"""
初始化git仓库
:param repo_url:
:param branch:
:return:
"""
if os.path.exists(self.local_path):
shutil.rmtree(self.local_path, ignore_errors=True)
os.makedirs(self.local_path)
print("clone empty-repo from remote:{0} to local:{1}".format(
repo_url, self.local_path))
print("please wait...\n")
self.repo = Repo.clone_from(repo_url,
to_path=self.local_path,
branch=branch,
bare=True)
def _clone(self,
local_dir: t.Optional[str] = None,
config_username: t.Optional[str] = None,
config_user_email: t.Optional[str] = None):
"""Clone the specified repository and recursively clone submodules.
This method raises a git.exec.GitCommandError if the repository does not exist.
"""
if local_dir:
self.local_dir = local_dir
else:
self.local_dir = tempfile.mkdtemp()
self.repo = Repo.clone_from(self.url,
self.local_dir,
multi_options=['--recurse-submodules'])
if config_username:
with self.repo.config_writer() as config:
config.set_value('user', 'email', config_user_email)
config.set_value('user', 'name', config_username)
def get_repo(repo_url: str, repo_path: Path):
if repo_path.exists():
repo = Repo(repo_path)
heads = Git().ls_remote(repo_url)
git_hash_length = 40
remote_last_commit = heads.split("\n")[0][:git_hash_length]
if remote_last_commit != repo.head.commit.hexsha:
update_repo = questionary.confirm(
"The remote repository has been updated."
"Do you want to update it?"
).ask()
if update_repo:
repo.remotes.origin.pull()
else:
# pyright: reportUnknownMemberType=false
repo = Repo.clone_from(repo_url, repo_path)
return repo
def _clone_repo(temp_dir, repo_url, verifyssl=True, ref='master'):
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
# Disable SSL cert checking if explictly asked
if not verifyssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, temp_dir)
# Try to match the reference to a commit hash, a tag, or "master"
gitref = DownloadGitRepoAction._get_gitref(repo, ref)
# Try to match the reference to a "vX.Y.Z" tag
if not gitref and re.match(SEMVER_REGEX, ref):
gitref = DownloadGitRepoAction._get_gitref(repo, "v%s" % ref)
# Try to match the reference to a branch name
if not gitref:
gitref = DownloadGitRepoAction._get_gitref(repo, "origin/%s" % ref)
# Giving up ¯\_(ツ)_/¯
if not gitref:
raise ValueError(
"\"%s\" is not a valid version, hash, tag, or branch in %s." % (ref, repo_url)
)
# We're trying to figure out which branch the ref is actually on,
# since there's no direct way to check for this in git-python.
branches = repo.git.branch('--color=never', '--all', '--contains', gitref.hexsha)
branches = branches.replace('*', '').split()
if 'master' not in branches:
branch = branches[0]
repo.git.checkout('--track', branches[0])
branch = repo.head.reference
else:
branch = 'master'
repo.git.checkout('-B', branch, gitref.hexsha)
return temp_dir
def _set_up_repo_state(local_base, git_remote, org, git_local,
repo_name, branch, commit, post_clone):
git_base = jpth(git_remote, org, repo_name)
if not os.path.exists(local_base):
repo = Repo.clone_from(git_base + '.git', local_base)
post_clone() # FIXME if this does not complete we need to warn or something, it causes errors
else:
repo = Repo(local_base)
nob = repo.active_branch
try:
nab = getBranch(repo, branch)
nab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
nab = repo.active_branch
repo.remote().pull() # make sure we are up to date
if commit != 'HEAD':
repo.git.checkout(commit)
return repo, nob
def setup(self):
git_opts = self._config
if git_opts['url'] is None:
raise Exception('Remote git URL not set.')
self._url = git_opts['url']
default_clone_dir = os.path.join(os.path.dirname(__file__), 'clones')
self._local_path = git_opts.get('local_clone_path', default_clone_dir)
self._poll_interval = git_opts.get('poll_interval', self._poll_interval)
if os.path.exists(self._local_path):
self._repo = Repo.init(self._local_path)
else:
try:
self._repo = Repo.clone_from(self._url, self._local_path)
except Exception:
self._logger.exception('Unable to clone remote repo from %s',
self._url)
raise
self._remote = self._repo.remote('origin')
def main():
# identify local packages in the environment and clone if they don't exist
for package in environment.local_packages:
pkg_path = locate_local_package(package)
prj_repo_name = f"{DEFAULT_BRANCH}-{PROJECT_NAME}"
# if package is local, switch to base branch "dev-init"
if pkg_path:
# switch branch to dev_init
repo = Repo(pkg_path)
if repo.active_branch.name != BASE_BRANCH:
repo.git.checkout(BASE_BRANCH)
repo.git.pull()
# otherwise clone the package from github
else:
print(f'Cloning {package}')
repo = Repo.clone_from(url=f'{RP_GITHUB_URL}/{package}', to_path=str(pkg_path), branch=BASE_BRANCH)
# check if the project branch exists, if not create it
if not check_branch(pkg_path, prj_repo_name):
repo.create_head(prj_repo_name)
# check out the project branch and push up to github
repo.git.checkout(prj_repo_name)
repo.git.push()
def clean_local_git(clean_now, git_local, remote_url):
#判断目录是否存在
if not os.path.exists(git_local):
logging.info('git_local[%s] not exist, no need to clean.', git_local)
return
#判断是否为清理时间(凌晨4点-24点)
if not time_in_range(time(4, 0), time(23, 0)) and not clean_now:
logging.info('git_local[%s] now[%s], not in clean time range[4:00-23:00].', git_local, datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
return
#判断今天是否已经清理过
if is_already_cleaned(git_local, clean_now):
logging.info('git_local[%s] is already cleaned, just continue.', git_local)
return
#清理
git_local_new = git_local + '.new'
if os.path.exists(git_local_new):
logging.info('rm dir[%s].', git_local_new)
rm_dir(git_local_new)
logging.info('git clone from [%s] to [%s], and depth=1.', remote_url, git_local_new)
repo_clone = Repo.clone_from(remote_url, git_local_new, depth=1)
if not repo_clone:
logging.error(
"clone repo from {0} to {1}failed.".format(remote_url, git_local_new))
logging.info('========run stop========\n\n')
sys.exit(1)
git_local_bak = git_local + '.bak'
if os.path.exists(git_local_bak):
logging.info('rm dir[%s].', git_local_bak)
rm_dir(git_local_bak)
logging.info('mv dir[%s] to dir[%s].', git_local, git_local_bak)
shutil.move(git_local, git_local_bak)
logging.info('mv dir[%s] to dir[%s].', git_local_new, git_local)
shutil.move(git_local_new, git_local)
import os
from git.repo import Repo
giturl = 'https://github.com/Andy-Jen/openCV_ws'
file_path = 'gitserver'
local_path = os.path.join(file_path)
if os.path.exists(file_path):
print('Find git server, check version...')
repo = Repo(local_path)
for tag in repo.tags:
print(tag.name)
else:
print('Nofind git server, Git clone for giturl...')
Repo.clone_from(giturl, to_path=local_path, branch='master')
def clone_repo(self, name, path):
Repo.clone_from("{}@{}:{}/{}.git".format(self.user,self.name,self.root,name),path)
def scigraph_build(zip_location,
git_remote,
org,
git_local,
branch,
commit,
clean=False,
check_built=False,
cleanup_later=False,
quiet=False):
COMMIT_LOG = 'last-built-commit.log'
repo_name = 'SciGraph'
remote = jpth(git_remote, org, repo_name)
local = jpth(git_local, repo_name)
commit_log_path = jpth(local, COMMIT_LOG)
if not os.path.exists(local):
repo = Repo.clone_from(remote + '.git', local)
elif not Path(local, '.git').exists():
repo = Repo.clone_from(remote + '.git', local)
else:
repo = Repo(local)
if not os.path.exists(commit_log_path):
last_commit = None
else:
with open(commit_log_path, 'rt') as f:
last_commit = f.read().strip()
sob = repo.active_branch
try:
sab = getBranch(repo, branch)
sab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
sab = repo.active_branch
repo.remote().pull()
if commit != 'HEAD':
repo.git.checkout(commit)
scigraph_commit = repo.head.object.hexsha
scigraph_commit_short = scigraph_commit[:COMMIT_HASH_HEAD_LEN]
bin_location = zip_location / 'bin'
os.environ['PATH'] = bin_location.as_posix() + ':' + os.environ.get(
'PATH', '')
if not bin_location.exists():
bin_location.mkdir()
# hack to make the scigraph-load we are about to create available as a command
# so that it matches the usual scigraph-load behavior
def zip_name(wild=False):
return (repo_name + '-' + branch + '-services' + '-' +
('*' if wild else TODAY()) + '-' + scigraph_commit_short +
'.zip')
def reset_state(original_branch=sob):
original_branch.checkout()
with execute_regardless(
reset_state, only_exception=cleanup_later
): # FIXME this fails when we need to load the graph if we start on master :/
# main
if scigraph_commit != last_commit or clean:
print('SciGraph not built at commit', commit, 'last built at',
last_commit)
quiet = '--quiet ' if quiet else ''
build_command = (
'cd ' + local + f';export HASH={scigraph_commit_short}'
';sed -i "/<name>SciGraph<\/name>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" pom.xml'
';sed -i "/<artifactId>scigraph<\/artifactId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-analysis/pom.xml'
';sed -i "/<groupId>io.scigraph<\/groupId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-core/pom.xml'
';sed -i "/<artifactId>scigraph<\/artifactId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-entity/pom.xml'
';sed -i "/<groupId>io.scigraph<\/groupId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-services/pom.xml'
f'; mvn {quiet}clean -DskipTests -DskipITs install'
'; cd SciGraph-services'
f'; mvn {quiet}-DskipTests -DskipITs package')
if check_built:
print('SciGraph has not been built.')
raise NotBuiltError('SciGraph has not been built.')
out = os.system(build_command)
print(out)
if out:
scigraph_commit = 'FAILURE'
with open(commit_log_path, 'wt') as f:
f.write(scigraph_commit)
else:
print('SciGraph already built at commit', scigraph_commit)
wildcard = jpth(zip_location, zip_name(wild=True))
try:
services_zip = glob(wildcard)[
0] # this will error if the zip was moved
return scigraph_commit, services_zip, reset_state
except IndexError:
pass # we need to copy the zip out again
# services zip
zip_filename = f'scigraph-services-{scigraph_commit_short}.zip'
services_zip_temp = Path(local, 'SciGraph-services', 'target',
zip_filename)
services_zip = jpth(zip_location, zip_name())
shutil.copy(services_zip_temp, services_zip)
core_jar = Path(
local, 'SciGraph-core', 'target',
f'scigraph-core-{scigraph_commit_short}-jar-with-dependencies.jar')
scigraph_load = f'''#!/usr/bin/env sh
/usr/bin/java \\
-cp "{core_jar.as_posix()}" \\
io.scigraph.owlapi.loader.BatchOwlLoader $@'''
slf = bin_location / 'scigraph-load'
with open(slf, 'wt') as f:
f.write(scigraph_load)
os.chmod(slf, 0o0755)
return scigraph_commit, services_zip, reset_state
def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'):
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
os.environ['GIT_ASKPASS'] = '******'
# Disable SSL cert checking if explictly asked
if not verify_ssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, temp_dir)
is_local_repo = repo_url.startswith('file://')
try:
active_branch = repo.active_branch
except TypeError as e:
if is_local_repo:
active_branch = None
else:
raise e
# Special case for local git repos - we allow users to install from repos which are checked out
# at a specific commit (aka detached HEAD)
if is_local_repo and not active_branch and not ref:
LOG.debug(
'Installing pack from git repo on disk, skipping branch checkout')
return temp_dir
use_branch = False
# Special case when a default repo branch is not "master"
# No ref provided so we just use a default active branch
if (not ref or ref == active_branch.name
) and repo.active_branch.object == repo.head.commit:
gitref = repo.active_branch.object
else:
# Try to match the reference to a branch name (i.e. "master")
gitref = get_gitref(repo, 'origin/%s' % ref)
if gitref:
use_branch = True
# Try to match the reference to a commit hash, a tag, or "master"
if not gitref:
gitref = get_gitref(repo, ref)
# Try to match the reference to a "vX.Y.Z" tag
if not gitref and re.match(PACK_VERSION_REGEX, ref):
gitref = get_gitref(repo, 'v%s' % ref)
# Giving up ¯\_(ツ)_/¯
if not gitref:
format_values = [ref, repo_url]
msg = '"%s" is not a valid version, hash, tag or branch in %s.'
valid_versions = get_valid_versions_for_repo(repo=repo)
if len(valid_versions) >= 1:
valid_versions_string = ', '.join(valid_versions)
msg += ' Available versions are: %s.'
format_values.append(valid_versions_string)
raise ValueError(msg % tuple(format_values))
# We're trying to figure out which branch the ref is actually on,
# since there's no direct way to check for this in git-python.
branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member
branches = branches.replace('*', '').split()
if active_branch.name not in branches or use_branch:
branch = 'origin/%s' % ref if use_branch else branches[0]
short_branch = ref if use_branch else branches[0].split('/')[-1]
repo.git.checkout('-b', short_branch, branch)
branch = repo.head.reference
else:
branch = repo.active_branch.name
repo.git.checkout(gitref.hexsha) # pylint: disable=no-member
repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member
repo.git.checkout(branch)
return temp_dir
def clone_repo(temp_dir, repo_url, verify_ssl=True, ref='master'):
# Switch to non-interactive mode
os.environ['GIT_TERMINAL_PROMPT'] = '0'
os.environ['GIT_ASKPASS'] = '******'
# Disable SSL cert checking if explictly asked
if not verify_ssl:
os.environ['GIT_SSL_NO_VERIFY'] = 'true'
# Clone the repo from git; we don't use shallow copying
# because we want the user to work with the repo in the
# future.
repo = Repo.clone_from(repo_url, temp_dir)
is_local_repo = repo_url.startswith('file://')
try:
active_branch = repo.active_branch
except TypeError as e:
if is_local_repo:
active_branch = None
else:
raise e
# Special case for local git repos - we allow users to install from repos which are checked out
# at a specific commit (aka detached HEAD)
if is_local_repo and not active_branch and not ref:
LOG.debug('Installing pack from git repo on disk, skipping branch checkout')
return temp_dir
use_branch = False
# Special case when a default repo branch is not "master"
# No ref provided so we just use a default active branch
if (not ref or ref == active_branch.name) and repo.active_branch.object == repo.head.commit:
gitref = repo.active_branch.object
else:
# Try to match the reference to a branch name (i.e. "master")
gitref = get_gitref(repo, 'origin/%s' % ref)
if gitref:
use_branch = True
# Try to match the reference to a commit hash, a tag, or "master"
if not gitref:
gitref = get_gitref(repo, ref)
# Try to match the reference to a "vX.Y.Z" tag
if not gitref and re.match(PACK_VERSION_REGEX, ref):
gitref = get_gitref(repo, 'v%s' % ref)
# Giving up ¯\_(ツ)_/¯
if not gitref:
format_values = [ref, repo_url]
msg = '"%s" is not a valid version, hash, tag or branch in %s.'
valid_versions = get_valid_versions_for_repo(repo=repo)
if len(valid_versions) >= 1:
valid_versions_string = ', '.join(valid_versions)
msg += ' Available versions are: %s.'
format_values.append(valid_versions_string)
raise ValueError(msg % tuple(format_values))
# We're trying to figure out which branch the ref is actually on,
# since there's no direct way to check for this in git-python.
branches = repo.git.branch('-a', '--contains', gitref.hexsha) # pylint: disable=no-member
branches = branches.replace('*', '').split()
if active_branch.name not in branches or use_branch:
branch = 'origin/%s' % ref if use_branch else branches[0]
short_branch = ref if use_branch else branches[0].split('/')[-1]
repo.git.checkout('-b', short_branch, branch)
branch = repo.head.reference
else:
branch = repo.active_branch.name
repo.git.checkout(gitref.hexsha) # pylint: disable=no-member
repo.git.branch('-f', branch, gitref.hexsha) # pylint: disable=no-member
repo.git.checkout(branch)
return temp_dir
def scigraph_build(zip_location,
git_remote,
org,
git_local,
branch,
commit,
clean=False,
check_built=False,
cleanup_later=False):
COMMIT_LOG = 'last-built-commit.log'
repo_name = 'SciGraph'
remote = jpth(git_remote, org, repo_name)
local = jpth(git_local, repo_name)
commit_log_path = jpth(local, COMMIT_LOG)
load_base = ('cd {}; '.format(jpth(local, 'SciGraph-core')) +
'mvn exec:java '
'-Dexec.mainClass="io.scigraph.owlapi.loader.BatchOwlLoader" '
'-Dexec.args="-c {config_path}"')
if not os.path.exists(local):
repo = Repo.clone_from(remote + '.git', local)
else:
repo = Repo(local)
if not os.path.exists(commit_log_path):
last_commit = None
else:
with open(commit_log_path, 'rt') as f:
last_commit = f.read().strip()
sob = repo.active_branch
try:
sab = getBranch(repo, branch)
sab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
sab = repo.active_branch
repo.remote().pull()
if commit != 'HEAD':
repo.git.checkout(commit)
scigraph_commit = repo.head.object.hexsha
def zip_name(wild=False):
return (repo_name + '-' + branch + '-services' + '-' +
('*' if wild else TODAY()) + '-' +
scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '.zip')
def reset_state(original_branch=sob):
original_branch.checkout()
with execute_regardless(
reset_state, only_exception=cleanup_later
): # FIXME this fails when we need to load the graph if we start on master :/
# main
if scigraph_commit != last_commit or clean:
print('SciGraph not built at commit', commit, 'last built at',
last_commit)
build_command = ('cd ' + local +
'; mvn clean -DskipTests -DskipITs install'
'; cd SciGraph-services'
'; mvn -DskipTests -DskipITs package')
if check_built:
print('SciGraph has not been built.')
raise NotBuiltError('SciGraph has not been built.')
out = os.system(build_command)
print(out)
if out:
scigraph_commit = 'FAILURE'
with open(commit_log_path, 'wt') as f:
f.write(scigraph_commit)
else:
print('SciGraph already built at commit', scigraph_commit)
wildcard = jpth(zip_location, zip_name(wild=True))
try:
services_zip = glob(wildcard)[
0] # this will error if the zip was moved
return scigraph_commit, load_base, services_zip, reset_state
except IndexError:
pass # we need to copy the zip out again
# services zip
zip_filename = 'scigraph-services-*-SNAPSHOT.zip'
services_zip_temp = glob(
jpth(local, 'SciGraph-services', 'target', zip_filename))[0]
services_zip = jpth(zip_location, zip_name())
shutil.copy(services_zip_temp, services_zip)
return scigraph_commit, load_base, services_zip, reset_state
def __init__(self,
zip_location,
git_remote,
org,
git_local,
repo_name,
branch,
commit,
remote_base,
load_base,
graphload_config,
patch_config,
patch,
scigraph_commit,
post_clone=lambda: None,
check_built=False):
local_base = jpth(git_local, repo_name)
git_base = jpth(git_remote, org, repo_name)
if not os.path.exists(local_base):
repo = Repo.clone_from(git_base + '.git', local_base)
post_clone(
) # FIXME if this does not complete we need to warn or something, it causes errors
else:
repo = Repo(local_base)
nob = repo.active_branch
try:
nab = getBranch(repo, branch)
nab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
nab = repo.active_branch
repo.remote().pull() # make sure we are up to date
if commit != 'HEAD':
repo.git.checkout(commit)
# TODO consider dumping metadata in a file in the folder too?
def folder_name(scigraph_commit, wild=False):
ontology_commit = repo.head.object.hexsha[:COMMIT_HASH_HEAD_LEN]
return (repo_name + '-' + branch + '-graph' + '-' +
('*' if wild else TODAY()) + '-' +
scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '-' +
ontology_commit)
def make_folder_zip(wild=False):
folder = folder_name(scigraph_commit, wild)
graph_path = jpth(zip_location, folder)
zip_path = graph_path + '.zip'
if wild:
return graph_path, zip_path
zip_name = os.path.basename(zip_path)
zip_dir = os.path.dirname(zip_path)
zip_command = ' '.join(
('cd', zip_dir, ';', 'zip -r', zip_name, folder))
return graph_path, zip_path, zip_command
graph_path, zip_path, zip_command = make_folder_zip()
wild_graph_path, wild_zip_path = make_folder_zip(wild=True)
(config, config_path,
ontologies) = self.make_graphload_config(graphload_config, graph_path,
remote_base, local_base,
zip_location)
load_command = load_base.format(
config_path=config_path) # 'exit 1' to test
print(load_command)
# replace raw github imports with ontology.neuinfor iris to simplify import chain
fix_imports = "find " + local_base + " -name '*.ttl' -exec sed -i 's/<http.\+\/ttl\//<http:\/\/ontology.neuinfo.org\/NIF\/ttl\//' {} \;"
os.system(fix_imports)
def reset_state(original_branch=nob):
repo.git.checkout('--', local_base)
original_branch.checkout()
with execute_regardless(
reset_state
): # FIXME start this immediately after we obtain nob?
# main
if patch:
# FIXME TODO XXX does scigraph load from the catalog!??!??
# because it seems like doid loads correctly without using local_versions
# which would be cool, if confusing
local_versions = tuple(do_patch(patch_config, local_base))
else:
local_versions = tuple()
itrips = local_imports(
remote_base,
local_base,
ontologies,
local_versions=local_versions,
dobig=True) # SciGraph doesn't support catalog.xml
catalog = make_catalog(itrips)
with open(Path(local_base, 'catalog.xml'), 'wt') as f:
f.write(catalog)
maybe_zip_path = glob(wild_zip_path)
if not maybe_zip_path:
if check_built:
print('The graph has not been loaded.')
raise NotBuiltError('The graph has not been loaded.')
failure = os.system(load_command)
if failure:
if os.path.exists(graph_path):
shutil.rmtree(graph_path)
else:
os.rename(
config_path, # save the config for eaiser debugging
jpth(graph_path, os.path.basename(config_path)))
failure = os.system(zip_command) # graphload zip
else:
zip_path = maybe_zip_path[0] # this way we get the actual date
print('Graph already loaded at', zip_path)
# this needs to be run when the branch is checked out
# FIXME might be worth adding this to the load config?
self.ontologies = [
get_iri(load_header(rec['url']))
for rec in config['ontologies']
]
self.zip_path = zip_path
self.itrips = itrips
self.config = config
