def pull_request(pathList):
addList, deleteList = [], []
for path in pathList:
repo = Repo(path)
# 检查版本库是否为空
if repo.bare:
return None
# 获取远程默认版本库 origin
remote = repo.remote()
try:
# 从远程版本库拉取分支
remote.pull()
except Exception as e:
print(path)
print(str(e))
git = repo.git
# 读取本地版本库的信息
strList = git.log('--numstat').split()
# 需要计算的文件
file = 'Units.cpp'
# 运行前需要设置时间
add, delete = count_add_delete(strList, file, 'Jun',10, 'Jun', 16)
addList.append(add)
deleteList.append(delete)
print('add:', addList)
print('delete:', deleteList)
write_excel(addList, deleteList)
def on_get(self, req, res):
repoPath = './IntallPlist' #本地仓库路径
result = 'success' #默认返回success
msg = []
repo = None
# 判断本地是否有仓库
if os.path.exists(repoPath):
repo = Repo(repoPath)
msg.append('已有仓库')
else:
repo = Repo.clone_from(
url='[email protected]:hillyoung/TestPlist.git', to_path=repoPath)
msg.append('不存在仓库')
git = repo.git
git.checkout('*')
remote = repo.remote()
remote.pull()
msg.append('完成拉取远程仓库代码')
# 判断是否传入了文件名
name = req.params.get('name', None)
if name is None:
msg.append('传入无效参数')
result = 'fail'
else:
msg.append('传入应用名：' + name)
# 构造plist文件内容及相关控制逻辑
directoryPath = repoPath + '/' + name
if os.path.exists(directoryPath) != True:
os.mkdir(directoryPath)
filePath = directoryPath + '/install.plist' #plist文件相对路径
rContent = None #用于缓存仓库中plist文件的内容
flag = 'w' #文件的打开访问，默认以写入方式打开
if os.path.exists(filePath): #判断仓库中是否存在plist文件
flag = 'r+' #如果plist文件存在，则以读写的方式打开
msg.append('存在plist文件')
content = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"><plist version="1.0"><dict><key>items</key><array><dict><key>assets</key><array><dict><key>kind</key><string>software-package</string><key>url</key><string>https://raw.githubusercontent.com/hillyoung/AppInstaller/master/app/' + name + '/LiemsMobileEnterprise.ipa</string></dict></array><key>metadata</key><dict><key>bundle-identifier</key> <string>net.luculent.liems.hhgs</string><key>bundle-version</key><string>1.0.1</string><key>kind</key><string>software</string><key>title</key><string>测试安装</string></dict></dict></array></dict></plist>'
with open(filePath, flag) as flie_object: #写入plist
if flag == 'r+':
rContent = flie_object.read()
flie_object.seek(0)
msg.append(rContent)
if rContent != content: #写入plist内容
flie_object.write(content)
flie_object.close()
#提交plist文件
git.add('*')
git.commit('-m', '添加' + name)
remote.push()
res.body = json.dumps({'result': result, 'msg': msg})
res.status = falcon.HTTP_200 # This is the default status
res.append_header('Access-Control-Allow-Origin', '*')
def getCommits(repo_name):
shas = set()
commits_list = []
repo = Repo(os.path.join(work_dir, repo_name))
for b in repo.remote().fetch():
if '/' not in b.name:
continue
print("start to check branch {} of {}".format(b.name, repo_name))
branch_name = b.name.split('/')[1]
repo.git.checkout('-B', branch_name, b.name)
commits = list(repo.iter_commits())
for idx, commit in enumerate(commits):
if str(commit) in shas:
continue
else:
shas.add(str(commit))
commit_collection = {}
xml_cnt = 0
kot_jav_cnt = 0
file_list = list(commit.stats.files)
for file in file_list:
if len(file) > 4 and file[-4:] == '.xml':
xml_cnt += 1
elif len(file) > 3 and file[-3:] == '.kt' or len(
file) > 5 and file[-5:] == '.java':
kot_jav_cnt += 1
if xml_cnt >= 1 and kot_jav_cnt >= 1:
commit_collection["commit_id"] = str(commit)
commit_collection["commit_msg"] = str.strip(commit.message)
commit_collection["commit_time"] = str(
commit.committed_datetime)
commit_collection["committer_name"] = commit.author.name
commit_collection["committer_email"] = commit.author.email
diff_files = []
if not commit.parents:
continue
else:
for diff in commit.parents[0].diff(commit):
diff_file = {}
diff_file["file_path"] = diff.a_path
diff_file["change_type"] = diff.change_type
diff_file["lang"] = os.path.splitext(
diff.a_path)[1][1:]
diff_files.append(diff_file)
commit_collection["diff_files"] = diff_files
commit_collection["parent_commit_num"] = len(commit.parents)
commits_list.append(commit_collection)
repo_dump = json.dumps(commits_list, indent=2, ensure_ascii=False)
with open(os.path.join(stats_dir, repo_name + ".json"), "w") as commit_fd:
commit_fd.write(repo_dump)
print(repo_name + " done")
def _set_up_repo_state(local_base, git_remote, org, git_local,
repo_name, branch, commit, post_clone):
git_base = jpth(git_remote, org, repo_name)
if not os.path.exists(local_base):
repo = Repo.clone_from(git_base + '.git', local_base)
post_clone() # FIXME if this does not complete we need to warn or something, it causes errors
else:
repo = Repo(local_base)
nob = repo.active_branch
try:
nab = getBranch(repo, branch)
nab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
nab = repo.active_branch
repo.remote().pull() # make sure we are up to date
if commit != 'HEAD':
repo.git.checkout(commit)
return repo, nob
class GitRepository(object):
def __init__(self, local_path, repo_url, branch='master'):
self.local_path = local_path
self.repo_url = repo_url
self.repo = None
self.initial(branch)
def initial(self, branch):
if not os.path.exists(self.local_path):
os.makedirs(self.local_path)
git_local_path = os.path.join(self.local_path, '.git')
if not is_git_dir(git_local_path):
self.repo = Repo.clone_from(self.repo_url,
to_path=self.local_path,
branch=branch)
else:
self.repo = Repo(self.local_path)
def pull(self):
self.repo.git.pull()
def branches(self):
branches = self.repo.remote().refs
return [
item.remote_head for item in branches if item.remote_head not in [
'HEAD',
]
]
def commits(self):
commit_log = self.repo.git.log(
'--pretty={"commit":"%h","author":"%an","summary":"%s","date":"%cd"}',
max_count=50,
date='format:%Y-%m-%d %H:%M')
return [json.loads(item) for item in commit_log.split('\n')]
def tags(self):
return [tag.name for tag in self.repo.tags]
def change_to_branch(self, branch):
self.repo.git.checkout(branch)
def change_to_commit(self, branch, commit):
self.change_to_branch(branch=branch)
self.repo.git.reset('--hard', commit)
def change_to_tag(self, tag):
self.repo.git.checkout(tag)
pass
def scigraph_build(zip_location,
git_remote,
org,
git_local,
branch,
commit,
clean=False,
check_built=False,
cleanup_later=False,
quiet=False):
COMMIT_LOG = 'last-built-commit.log'
repo_name = 'SciGraph'
remote = jpth(git_remote, org, repo_name)
local = jpth(git_local, repo_name)
commit_log_path = jpth(local, COMMIT_LOG)
if not os.path.exists(local):
repo = Repo.clone_from(remote + '.git', local)
elif not Path(local, '.git').exists():
repo = Repo.clone_from(remote + '.git', local)
else:
repo = Repo(local)
if not os.path.exists(commit_log_path):
last_commit = None
else:
with open(commit_log_path, 'rt') as f:
last_commit = f.read().strip()
sob = repo.active_branch
try:
sab = getBranch(repo, branch)
sab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
sab = repo.active_branch
repo.remote().pull()
if commit != 'HEAD':
repo.git.checkout(commit)
scigraph_commit = repo.head.object.hexsha
scigraph_commit_short = scigraph_commit[:COMMIT_HASH_HEAD_LEN]
bin_location = zip_location / 'bin'
os.environ['PATH'] = bin_location.as_posix() + ':' + os.environ.get(
'PATH', '')
if not bin_location.exists():
bin_location.mkdir()
# hack to make the scigraph-load we are about to create available as a command
# so that it matches the usual scigraph-load behavior
def zip_name(wild=False):
return (repo_name + '-' + branch + '-services' + '-' +
('*' if wild else TODAY()) + '-' + scigraph_commit_short +
'.zip')
def reset_state(original_branch=sob):
original_branch.checkout()
with execute_regardless(
reset_state, only_exception=cleanup_later
): # FIXME this fails when we need to load the graph if we start on master :/
# main
if scigraph_commit != last_commit or clean:
print('SciGraph not built at commit', commit, 'last built at',
last_commit)
quiet = '--quiet ' if quiet else ''
build_command = (
'cd ' + local + f';export HASH={scigraph_commit_short}'
';sed -i "/<name>SciGraph<\/name>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" pom.xml'
';sed -i "/<artifactId>scigraph<\/artifactId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-analysis/pom.xml'
';sed -i "/<groupId>io.scigraph<\/groupId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-core/pom.xml'
';sed -i "/<artifactId>scigraph<\/artifactId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-entity/pom.xml'
';sed -i "/<groupId>io.scigraph<\/groupId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-services/pom.xml'
f'; mvn {quiet}clean -DskipTests -DskipITs install'
'; cd SciGraph-services'
f'; mvn {quiet}-DskipTests -DskipITs package')
if check_built:
print('SciGraph has not been built.')
raise NotBuiltError('SciGraph has not been built.')
out = os.system(build_command)
print(out)
if out:
scigraph_commit = 'FAILURE'
with open(commit_log_path, 'wt') as f:
f.write(scigraph_commit)
else:
print('SciGraph already built at commit', scigraph_commit)
wildcard = jpth(zip_location, zip_name(wild=True))
try:
services_zip = glob(wildcard)[
0] # this will error if the zip was moved
return scigraph_commit, services_zip, reset_state
except IndexError:
pass # we need to copy the zip out again
# services zip
zip_filename = f'scigraph-services-{scigraph_commit_short}.zip'
services_zip_temp = Path(local, 'SciGraph-services', 'target',
zip_filename)
services_zip = jpth(zip_location, zip_name())
shutil.copy(services_zip_temp, services_zip)
core_jar = Path(
local, 'SciGraph-core', 'target',
f'scigraph-core-{scigraph_commit_short}-jar-with-dependencies.jar')
scigraph_load = f'''#!/usr/bin/env sh
/usr/bin/java \\
-cp "{core_jar.as_posix()}" \\
io.scigraph.owlapi.loader.BatchOwlLoader $@'''
slf = bin_location / 'scigraph-load'
with open(slf, 'wt') as f:
f.write(scigraph_load)
os.chmod(slf, 0o0755)
return scigraph_commit, services_zip, reset_state
def scigraph_build(zip_location,
git_remote,
org,
git_local,
branch,
commit,
clean=False,
check_built=False,
cleanup_later=False):
COMMIT_LOG = 'last-built-commit.log'
repo_name = 'SciGraph'
remote = jpth(git_remote, org, repo_name)
local = jpth(git_local, repo_name)
commit_log_path = jpth(local, COMMIT_LOG)
load_base = ('cd {}; '.format(jpth(local, 'SciGraph-core')) +
'mvn exec:java '
'-Dexec.mainClass="io.scigraph.owlapi.loader.BatchOwlLoader" '
'-Dexec.args="-c {config_path}"')
if not os.path.exists(local):
repo = Repo.clone_from(remote + '.git', local)
else:
repo = Repo(local)
if not os.path.exists(commit_log_path):
last_commit = None
else:
with open(commit_log_path, 'rt') as f:
last_commit = f.read().strip()
sob = repo.active_branch
try:
sab = getBranch(repo, branch)
sab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
sab = repo.active_branch
repo.remote().pull()
if commit != 'HEAD':
repo.git.checkout(commit)
scigraph_commit = repo.head.object.hexsha
def zip_name(wild=False):
return (repo_name + '-' + branch + '-services' + '-' +
('*' if wild else TODAY()) + '-' +
scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '.zip')
def reset_state(original_branch=sob):
original_branch.checkout()
with execute_regardless(
reset_state, only_exception=cleanup_later
): # FIXME this fails when we need to load the graph if we start on master :/
# main
if scigraph_commit != last_commit or clean:
print('SciGraph not built at commit', commit, 'last built at',
last_commit)
build_command = ('cd ' + local +
'; mvn clean -DskipTests -DskipITs install'
'; cd SciGraph-services'
'; mvn -DskipTests -DskipITs package')
if check_built:
print('SciGraph has not been built.')
raise NotBuiltError('SciGraph has not been built.')
out = os.system(build_command)
print(out)
if out:
scigraph_commit = 'FAILURE'
with open(commit_log_path, 'wt') as f:
f.write(scigraph_commit)
else:
print('SciGraph already built at commit', scigraph_commit)
wildcard = jpth(zip_location, zip_name(wild=True))
try:
services_zip = glob(wildcard)[
0] # this will error if the zip was moved
return scigraph_commit, load_base, services_zip, reset_state
except IndexError:
pass # we need to copy the zip out again
# services zip
zip_filename = 'scigraph-services-*-SNAPSHOT.zip'
services_zip_temp = glob(
jpth(local, 'SciGraph-services', 'target', zip_filename))[0]
services_zip = jpth(zip_location, zip_name())
shutil.copy(services_zip_temp, services_zip)
return scigraph_commit, load_base, services_zip, reset_state
def __init__(self,
zip_location,
git_remote,
org,
git_local,
repo_name,
branch,
commit,
remote_base,
load_base,
graphload_config,
patch_config,
patch,
scigraph_commit,
post_clone=lambda: None,
check_built=False):
local_base = jpth(git_local, repo_name)
git_base = jpth(git_remote, org, repo_name)
if not os.path.exists(local_base):
repo = Repo.clone_from(git_base + '.git', local_base)
post_clone(
) # FIXME if this does not complete we need to warn or something, it causes errors
else:
repo = Repo(local_base)
nob = repo.active_branch
try:
nab = getBranch(repo, branch)
nab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
nab = repo.active_branch
repo.remote().pull() # make sure we are up to date
if commit != 'HEAD':
repo.git.checkout(commit)
# TODO consider dumping metadata in a file in the folder too?
def folder_name(scigraph_commit, wild=False):
ontology_commit = repo.head.object.hexsha[:COMMIT_HASH_HEAD_LEN]
return (repo_name + '-' + branch + '-graph' + '-' +
('*' if wild else TODAY()) + '-' +
scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '-' +
ontology_commit)
def make_folder_zip(wild=False):
folder = folder_name(scigraph_commit, wild)
graph_path = jpth(zip_location, folder)
zip_path = graph_path + '.zip'
if wild:
return graph_path, zip_path
zip_name = os.path.basename(zip_path)
zip_dir = os.path.dirname(zip_path)
zip_command = ' '.join(
('cd', zip_dir, ';', 'zip -r', zip_name, folder))
return graph_path, zip_path, zip_command
graph_path, zip_path, zip_command = make_folder_zip()
wild_graph_path, wild_zip_path = make_folder_zip(wild=True)
(config, config_path,
ontologies) = self.make_graphload_config(graphload_config, graph_path,
remote_base, local_base,
zip_location)
load_command = load_base.format(
config_path=config_path) # 'exit 1' to test
print(load_command)
# replace raw github imports with ontology.neuinfor iris to simplify import chain
fix_imports = "find " + local_base + " -name '*.ttl' -exec sed -i 's/<http.\+\/ttl\//<http:\/\/ontology.neuinfo.org\/NIF\/ttl\//' {} \;"
os.system(fix_imports)
def reset_state(original_branch=nob):
repo.git.checkout('--', local_base)
original_branch.checkout()
with execute_regardless(
reset_state
): # FIXME start this immediately after we obtain nob?
# main
if patch:
# FIXME TODO XXX does scigraph load from the catalog!??!??
# because it seems like doid loads correctly without using local_versions
# which would be cool, if confusing
local_versions = tuple(do_patch(patch_config, local_base))
else:
local_versions = tuple()
itrips = local_imports(
remote_base,
local_base,
ontologies,
local_versions=local_versions,
dobig=True) # SciGraph doesn't support catalog.xml
catalog = make_catalog(itrips)
with open(Path(local_base, 'catalog.xml'), 'wt') as f:
f.write(catalog)
maybe_zip_path = glob(wild_zip_path)
if not maybe_zip_path:
if check_built:
print('The graph has not been loaded.')
raise NotBuiltError('The graph has not been loaded.')
failure = os.system(load_command)
if failure:
if os.path.exists(graph_path):
shutil.rmtree(graph_path)
else:
os.rename(
config_path, # save the config for eaiser debugging
jpth(graph_path, os.path.basename(config_path)))
failure = os.system(zip_command) # graphload zip
else:
zip_path = maybe_zip_path[0] # this way we get the actual date
print('Graph already loaded at', zip_path)
# this needs to be run when the branch is checked out
# FIXME might be worth adding this to the load config?
self.ontologies = [
get_iri(load_header(rec['url']))
for rec in config['ontologies']
]
self.zip_path = zip_path
self.itrips = itrips
self.config = config
def check(repo_obj):
global row
global commit_cnt
global changed_files_cnt
global all_commit_cnt
global all_changed_cnt
repo_name = repo_obj["owner"]["login"] + "-" + repo_obj["name"]
# print("="*8, "start to check repo ", repo_name, "="*8)
commit_total = 0
commit_cross = 0
percent = 0
shas = set()
repo = Repo(os.path.join(work_dir, repo_name))
for b in repo.remote().fetch():
if '/' not in b.name:
continue
print("start to check branch {} of {}".format(b.name, repo_name))
branch_name = b.name.split('/')[1]
repo.git.checkout('-B', branch_name, b.name)
commits = list(repo.iter_commits())
for idx, commit in enumerate(commits):
if str(commit) in shas:
continue
else:
shas.add(str(commit))
if len(commit.parents) > 1:
continue
commit_total += 1
xml_cnt = 0
kot_jav_cnt = 0
all_commit_cnt += 1
# if idx == len(commits)-1:
file_list = list(commit.stats.files)
all_changed_cnt += len(file_list)
for file in file_list:
if len(file) > 4 and file[-4:] == '.xml':
xml_cnt += 1
elif len(file) > 3 and file[-3:] == '.kt' or len(
file) > 5 and file[-5:] == '.java':
kot_jav_cnt += 1
if xml_cnt >= 1 and kot_jav_cnt >= 1:
commit_cnt += 1
changed_files_cnt += len(file_list)
commit_cross += 1
# break
# diff_index = commit.diff(commits[idx+1])
# for diff_item in diff_index:
# if diff_item.a_path[-4:] == '.xml':
# xml_cnt += 1
# elif diff_item.a_path[-3:] == '.kt' or diff_item.a_path[-5:] == '.java':
# kot_jav_cnt += 1
# if xml_cnt >= 1 and kot_jav_cnt >= 1:
# commit_cross += 1
percent = float(commit_cross) / commit_total
repo_name_full = repo_obj["full_name"]
res = "{} Total: {}, Cross: {}, Percent: {}".format(
repo_name_full, commit_total, commit_cross, percent)
print(res)
percent_coll.append((repo_obj["full_name"], percent * 100))
# print("="*8, "check repo ", repo_name, "completed", "="*8)
return commit_total, commit_cross, percent
class GitRepository(object):
"""
git仓库管理
"""
def __init__(self, local_path, repo_url, branch='master'):
self.local_path = local_path
self.repo_url = repo_url
self.repo = None
self.initial(repo_url, branch)
def initial(self, repo_url, branch):
"""
初始化git仓库
:param repo_url:
:param branch:
:return:
"""
if not os.path.exists(self.local_path):
os.makedirs(self.local_path)
git_local_path = os.path.join(self.local_path, '.git')
if not is_git_dir(git_local_path):
self.repo = Repo.clone_from(repo_url, to_path=self.local_path, branch=branch)
else:
self.repo = Repo(self.local_path)
def pull(self):
"""
从线上拉最新代码
:return:
"""
self.repo.git.pull()
def branches(self):
"""
获取所有分支
:return:
"""
branches = self.repo.remote().refs
return [item.remote_head for item in branches if item.remote_head not in ['HEAD', ]]
def commits(self):
"""
获取所有提交记录
:return:
"""
commit_log = self.repo.git.log('--pretty={"commit":"%h","author":"%an","summary":"%s","date":"%cd"}',
max_count=50,
date='format:%Y-%m-%d %H:%M')
log_list = commit_log.split("\n")
return [eval(item) for item in log_list]
def tags(self):
"""
获取所有tag
:return:
"""
return [tag.name for tag in self.repo.tags]
def change_to_branch(self, branch):
"""
切换分值
:param branch:
:return:
"""
self.repo.git.checkout(branch)
def change_to_commit(self, branch, commit):
"""
切换commit
:param branch:
:param commit:
:return:
"""
self.change_to_branch(branch=branch)
self.repo.git.reset('--hard', commit)
def change_to_tag(self, tag):
"""
切换tag
:param tag:
:return:
"""
self.repo.git.checkout(tag)
def check(repo_name):
# global commit_cnt
# global changed_files_cnt
# global all_commit_cnt
# global all_changed_cnt
global multi_lang
global other
shas = set()
repo = Repo(os.path.join(work_dir, repo_name))
for b in repo.remote().fetch():
if '/' not in b.name:
continue
print("start to check branch {} of {}".format(b.name, repo_name))
# branch_name = b.name.split('/')[1]
# repo.git.checkout('-B', branch_name, b.name)
commits = list(repo.iter_commits('remotes/' + b.name))
for idx, commit in enumerate(commits):
if str(commit) in shas:
continue
else:
shas.add(str(commit))
if len(commits[idx - 1].parents) > 1:
continue
if idx == 0:
continue
xml_cnt = 0
kot_jav_cnt = 0
# if idx == len(commits)-1:
file_list = list(commits[idx - 1].stats.files)
dir_set = set()
for file in file_list:
file = file.split(' => ')[-1]
dir = file.split('/')[0]
if dir == file:
dir_set.add('.')
else:
dir_set.add(dir)
if len(file) > 4 and file[-4:] == '.xml':
xml_cnt += 1
elif len(file) > 3 and file[-3:] == '.kt' or len(
file) > 5 and file[-5:] == '.java':
kot_jav_cnt += 1
dir_cnt = len(dir_set)
patch = repo.git.diff(commit.tree,
commits[idx - 1].tree).split('\n')
hunks_cnt = 0
added = 0
deleted = 0
for line in patch:
if len(line) >= 2 and line[0] == '@' and line[1] == '@':
hunks_cnt += 1
elif len(line) >= 1 and line[0] == '+' and (len(line) < 3
or line[1] != '+'
or line[2] != '+'):
added += 1
elif len(line) >= 1 and line[0] == '-' and (len(line) < 3
or line[1] != '-'
or line[2] != '-'):
deleted += 1
commit_type = ""
if xml_cnt >= 1 and kot_jav_cnt >= 1:
commit_type = "Multi-lang"
multi_lang += 1
else:
commit_type = "Other"
other += 1
with open(os.path.join(csv_dir, metric_type[0] + ".csv"),
'a',
newline="") as csv_fd:
csv_writer = csv.writer(csv_fd,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow([commit_type, len(file_list)])
if len(file_list) == 0:
print("!!" + str(commits[idx - 1]))
with open(os.path.join(csv_dir, metric_type[1] + ".csv"),
'a',
newline="") as csv_fd:
csv_writer = csv.writer(csv_fd,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow([commit_type, hunks_cnt])
with open(os.path.join(csv_dir, metric_type[2] + ".csv"),
'a',
newline="") as csv_fd:
csv_writer = csv.writer(csv_fd,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow([commit_type, added])
with open(os.path.join(csv_dir, metric_type[3] + ".csv"),
'a',
newline="") as csv_fd:
csv_writer = csv.writer(csv_fd,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow([commit_type, deleted])
with open(os.path.join(csv_dir, metric_type[4] + ".csv"),
'a',
newline="") as csv_fd:
csv_writer = csv.writer(csv_fd,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL)
csv_writer.writerow([commit_type, dir_cnt])
# break
# diff_index = commit.diff(commits[idx+1])
# for diff_item in diff_index:
# if diff_item.a_path[-4:] == '.xml':
# xml_cnt += 1
# elif diff_item.a_path[-3:] == '.kt' or diff_item.a_path[-5:] == '.java':
# kot_jav_cnt += 1
# if xml_cnt >= 1 and kot_jav_cnt >= 1:
# commit_cross += 1
print("repo {} done".format(repo_name))
def getQRCommits(repo_name):
global QRCommits_cnt
repo_cnt = 0
commits_list = []
shas = set()
repo = Repo(os.path.join(work_dir, repo_name))
for b in repo.remote().fetch():
if '/' not in b.name:
continue
print("start to check branch {} of {}".format(b.name, repo_name))
branch_name = b.name.split('/')[1]
repo.git.checkout('-B', branch_name, b.name)
commits = list(repo.iter_commits(branch_name))
for idx, commit in enumerate(commits):
if str(commit) in shas:
continue
else:
shas.add(str(commit))
commit_collection = {}
xml_cnt = 0
kot_jav_cnt = 0
file_list = list(commit.stats.files)
for file in file_list:
if len(file) > 4 and file[-4:] == '.xml':
xml_cnt += 1
elif len(file) > 3 and file[-3:] == '.kt' or len(
file) > 5 and file[-5:] == '.java':
kot_jav_cnt += 1
if xml_cnt >= 1 and kot_jav_cnt >= 1:
if idx != 0:
valid = False
for f in list(commits[idx - 1].stats.files):
if '.xml' in f or '.kt' in f or '.java' in f:
valid = True
break
if not valid:
continue
time_next_commit = commits[
idx - 1].committed_datetime - commit.committed_datetime
if time_next_commit.total_seconds() / 60 <= 30 and len(
commit.parents) < 2 and len(
commits[idx - 1].parents) < 2:
QRCommits_cnt += 1
repo_cnt += 1
commit_collection["commit_id"] = str(commit)
commit_collection["commit_time"] = str(
commit.committed_datetime)
commit_collection["commit_msg"] = str.strip(
commit.message)
commit_collection["remedy_id"] = str(commits[idx - 1])
commit_collection["remedy_time"] = str(
commits[idx - 1].committed_datetime)
commit_collection["remedy_msg"] = str.strip(
commits[idx - 1].message)
commits_list.append(commit_collection)
# diff_files = []
# if not commit.parents:
# continue
# else:
# for diff in commit.diff(commit.parents[0]):
# diff_file = {}
# diff_file["file_path"] = diff.a_path
# diff_file["change_type"] = diff.change_type
# diff_file["lang"] = os.path.splitext(diff.a_path)[1][1:]
# diff_files.append(diff_file)
# commit_collection["diff_files"] = diff_files
# commit_collection["parent_commit_num"] = len(commit.parents)
repo_dump = json.dumps(commits_list, indent=2, ensure_ascii=False)
with open(os.path.join(stats_dir, repo_name + "_" + str(repo_cnt)),
"w") as commit_fd:
commit_fd.write(repo_dump)
print(repo_name + " done with {} commits".format(repo_cnt))
class GitRepository(object):
"""
实现Git仓库自动管理
"""
def __init__(self, local_path, repo_url, branch='main'):
self.local_path = local_path
self.repo_url = repo_url
self.repo = None
self.initial(repo_url, branch)
# self.commitMessage = commitMessage
def initial(self, repo_url, branch):
"""
初始化git仓库
"""
if not os.path.exists(self.local_path):
os.makedirs(self.local_path)
git_local_path = os.path.join(self.local_path, '.git')
if not is_git_dir(git_local_path):
self.repo = Repo.clone_from(repo_url,
to_path=self.local_path,
branch=branch)
else:
self.repo = Repo(self.local_path)
def pull(self):
"""
从线上拉最新代码
:return:
"""
self.repo.git.pull()
def branches(self):
"""
获取所有分支
:return:
"""
branches = self.repo.remote().refs
return [
item.remote_head for item in branches if item.remote_head not in [
'HEAD',
]
]
# def commits_log(self):
# """
# 获取所有提交记录
# :return:
# """
# commit_log = Repo.git.log('--pretty={"commit":"%h","author":"%an","summary":"%s","date":"%cd"}', max_count=50,
# date='format:%Y-%m-%d %H:%M')
# log_list = commit_log.split("\n")
# return [eval(item) for item in log_list]
def add(self):
#cmd = "git add ." + self.local_path
cmd = "git add ."
process = subprocess.Popen(cmd, shell=True)
process.wait()
returnCode = process.returncode
if returnCode != 0:
print(" add returnCode", returnCode)
else:
print('add success')
self.commit()
def push(self):
cmd = "git push"
process = subprocess.Popen(cmd, shell=True)
process.wait()
returnCode = process.returncode
if returnCode != 0:
print("push returnCode", returnCode)
else:
print('push success')
# sendMessage({
# "fileName": "api文档 : \n\n已更新，请注意查看！ \n" +"\n更新信息: {}".format(
# commitMessage),
# "text": time.strftime("%Y/%m/%d %H:%M"),
# "error": False
# })
# pass
def commit(self):
msg = input("输入提交commit信息:")
commitMessage = time.strftime("%Y/%m/%d %H:%M") + '-' + msg
cmd = "git commit -m '{}'".format(commitMessage)
process = subprocess.Popen(cmd, shell=True)
process.wait()
self.push()
def tags(self):
"""
获取所有tag
:return:
"""
return [tag.name for tag in self.repo.tags]
def change_to_branch(self, branch):
"""
切换分支
:param branch:
:return:
"""
self.repo.git.checkout(branch)
def change_to_commit(self, branch, commit):
"""
切换commit
:param branch:
:param commit:
:return:
"""
self.change_to_branch(branch=branch)
self.repo.git.reset('--hard', commit)
def change_to_tag(self, tag):
"""
切换tag
:param tag:
:return:
"""
self.repo.git.checkout(tag)
def computeRepo(repo_name):
global issue_cnt
global chinese_mst_cnt
shas = set()
print(8 * '=', "start to handle repo ", repo_name, 8 * '=')
repo = Repo(os.path.join(repo_dir, repo_name))
for b in repo.remote().fetch():
if '/' not in b.name:
continue
print("start to check branch {} of {}".format(b.name, repo_name))
branch_name = b.name.split('/')[1]
repo.git.checkout('-B', branch_name, b.name)
commits = list(repo.iter_commits(branch_name))
for idx, commit in enumerate(commits):
if str(commit) in shas:
continue
else:
shas.add(str(commit))
xml_cnt = 0
kot_jav_cnt = 0
for file in list(commit.stats.files):
if len(file) > 4 and file[-4:] == '.xml':
xml_cnt += 1
elif len(file) > 3 and file[-3:] == '.kt' or len(
file) > 5 and file[-5:] == '.java':
kot_jav_cnt += 1
if xml_cnt >= 1 and kot_jav_cnt >= 1:
msg = commit.message
# msg = str.strip(
# translator.translate(msg,
# lang_tgt='en')).lower()
# print(msg)
if reg_issue.search(msg) and len(commit.parents) < 2:
msg = "fix " + msg
issue_cnt += 1
if (check_contain_chinese(msg)):
chinese_mst_cnt += 1
seg_list = jieba.cut(msg, cut_all=False)
cn = True
else:
seg_list = wordninja.split(msg.lower())
cn = False
for seg in seg_list:
if cn is True and check_contain_chinese(seg) is False:
seg_split_again = wordninja.split(seg)
for _seg in seg_split_again:
if len(
_seg
) > 1 and _seg != '\t' and _seg != '\n' and _seg != '\r\n' and _seg not in stop_words:
if _seg in word_bags:
word_bags[_seg] += 1
else:
word_bags[_seg] = 1
elif cn is True:
if len(
seg
) > 1 and seg != '\t' and seg != '\n' and seg != '\r\n' and seg not in stop_words:
if seg in word_bags:
word_bags[seg] += 1
else:
word_bags[seg] = 1
else:
if len(
seg
) > 1 and seg != '\t' and seg != '\n' and seg != '\r\n' and seg not in stop_words:
if seg in word_bags:
word_bags[seg] += 1
else:
word_bags[seg] = 1
print(8 * '=', "repo ", repo_name, "done", 8 * '=')
def repro_loader(zip_location,
git_remote,
org,
git_local,
repo_name,
branch,
commit,
remote_base,
load_base,
graphload_config,
scigraph_commit,
post_clone=lambda: None,
check_built=False):
local_base = jpth(git_local, repo_name)
git_base = jpth(git_remote, org, repo_name)
if not os.path.exists(local_base):
repo = Repo.clone_from(git_base + '.git', local_base)
post_clone(
) # FIXME if this does not complete we need to warn or something, it causes errors
else:
repo = Repo(local_base)
nob = repo.active_branch
try:
nab = getBranch(repo, branch)
nab.checkout()
except ValueError: # usually indicates a remote branch
repo.git.checkout(branch)
nab = repo.active_branch
repo.remote().pull() # make sure we are up to date
if commit != 'HEAD':
repo.git.checkout(commit)
# TODO consider dumping metadata in a file in the folder too?
def folder_name(scigraph_commit, wild=False):
ontology_commit = repo.head.object.hexsha[:COMMIT_HASH_HEAD_LEN]
return (repo_name + '-' + branch + '-graph' + '-' +
('*' if wild else TODAY) + '-' +
scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '-' + ontology_commit)
def make_folder_zip(wild=False):
folder = folder_name(scigraph_commit, wild)
graph_path = jpth(zip_location, folder)
zip_path = graph_path + '.zip'
if wild:
return graph_path, zip_path
zip_name = os.path.basename(zip_path)
zip_dir = os.path.dirname(zip_path)
zip_command = ' '.join(
('cd', zip_dir, ';', 'zip -r', zip_name, folder))
return graph_path, zip_path, zip_command
graph_path, zip_path, zip_command = make_folder_zip()
wild_graph_path, wild_zip_path = make_folder_zip(wild=True)
# config graphload.yaml from template
graphload_config_template = graphload_config + '.template'
with open(graphload_config_template, 'rt') as f:
config = yaml.load(f)
config['graphConfiguration']['location'] = graph_path
config['ontologies'] = [{
k: v.replace(remote_base, local_base) if k == 'url' else v
for k, v in ont.items()
} for ont in config['ontologies']]
config_path = jpth(zip_location, 'graphload-' + TODAY + '.yaml')
with open(config_path, 'wt') as f:
yaml.dump(config, f, default_flow_style=False)
ontologies = [ont['url'] for ont in config['ontologies']]
load_command = load_base.format(config_path=config_path)
print(load_command)
def reset_state(original_branch=nob):
original_branch.checkout()
# return to original state (reset --hard)
repo.head.reset(
index=True, working_tree=True
) # FIXME we need to not run anything if there are files added to staging
with execute_regardless(
reset_state): # FIXME start this immediately after we obtain nob?
# main
itrips = local_imports(
remote_base, local_base,
ontologies) # SciGraph doesn't support catalog.xml
maybe_zip_path = glob(wild_zip_path)
if not maybe_zip_path:
if check_built:
print('The graph has not been loaded.')
raise NotBuiltError('The graph has not been loaded.')
failure = os.system(load_command)
if failure:
if os.path.exists(graph_path):
shutil.rmtree(graph_path)
else:
os.rename(
config_path, # save the config for eaiser debugging
jpth(graph_path, os.path.basename(config_path)))
failure = os.system(zip_command) # graphload zip
else:
zip_path = maybe_zip_path[0] # this way we get the actual date
print('Graph already loaded at', zip_path)
return zip_path, itrips
