def pull_request(pathList): addList, deleteList = [], [] for path in pathList: repo = Repo(path) # 检查版本库是否为空 if repo.bare: return None # 获取远程默认版本库 origin remote = repo.remote() try: # 从远程版本库拉取分支 remote.pull() except Exception as e: print(path) print(str(e)) git = repo.git # 读取本地版本库的信息 strList = git.log('--numstat').split() # 需要计算的文件 file = 'Units.cpp' # 运行前需要设置时间 add, delete = count_add_delete(strList, file, 'Jun',10, 'Jun', 16) addList.append(add) deleteList.append(delete) print('add:', addList) print('delete:', deleteList) write_excel(addList, deleteList)
def on_get(self, req, res): repoPath = './IntallPlist' #本地仓库路径 result = 'success' #默认返回success msg = [] repo = None # 判断本地是否有仓库 if os.path.exists(repoPath): repo = Repo(repoPath) msg.append('已有仓库') else: repo = Repo.clone_from( url='[email protected]:hillyoung/TestPlist.git', to_path=repoPath) msg.append('不存在仓库') git = repo.git git.checkout('*') remote = repo.remote() remote.pull() msg.append('完成拉取远程仓库代码') # 判断是否传入了文件名 name = req.params.get('name', None) if name is None: msg.append('传入无效参数') result = 'fail' else: msg.append('传入应用名:' + name) # 构造plist文件内容及相关控制逻辑 directoryPath = repoPath + '/' + name if os.path.exists(directoryPath) != True: os.mkdir(directoryPath) filePath = directoryPath + '/install.plist' #plist文件相对路径 rContent = None #用于缓存仓库中plist文件的内容 flag = 'w' #文件的打开访问,默认以写入方式打开 if os.path.exists(filePath): #判断仓库中是否存在plist文件 flag = 'r+' #如果plist文件存在,则以读写的方式打开 msg.append('存在plist文件') content = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"><plist version="1.0"><dict><key>items</key><array><dict><key>assets</key><array><dict><key>kind</key><string>software-package</string><key>url</key><string>https://raw.githubusercontent.com/hillyoung/AppInstaller/master/app/' + name + '/LiemsMobileEnterprise.ipa</string></dict></array><key>metadata</key><dict><key>bundle-identifier</key> <string>net.luculent.liems.hhgs</string><key>bundle-version</key><string>1.0.1</string><key>kind</key><string>software</string><key>title</key><string>测试安装</string></dict></dict></array></dict></plist>' with open(filePath, flag) as flie_object: #写入plist if flag == 'r+': rContent = flie_object.read() flie_object.seek(0) msg.append(rContent) if rContent != content: #写入plist内容 flie_object.write(content) flie_object.close() #提交plist文件 git.add('*') git.commit('-m', '添加' + name) remote.push() res.body = json.dumps({'result': result, 'msg': msg}) res.status = falcon.HTTP_200 # This is the default status res.append_header('Access-Control-Allow-Origin', '*')
def getCommits(repo_name): shas = set() commits_list = [] repo = Repo(os.path.join(work_dir, repo_name)) for b in repo.remote().fetch(): if '/' not in b.name: continue print("start to check branch {} of {}".format(b.name, repo_name)) branch_name = b.name.split('/')[1] repo.git.checkout('-B', branch_name, b.name) commits = list(repo.iter_commits()) for idx, commit in enumerate(commits): if str(commit) in shas: continue else: shas.add(str(commit)) commit_collection = {} xml_cnt = 0 kot_jav_cnt = 0 file_list = list(commit.stats.files) for file in file_list: if len(file) > 4 and file[-4:] == '.xml': xml_cnt += 1 elif len(file) > 3 and file[-3:] == '.kt' or len( file) > 5 and file[-5:] == '.java': kot_jav_cnt += 1 if xml_cnt >= 1 and kot_jav_cnt >= 1: commit_collection["commit_id"] = str(commit) commit_collection["commit_msg"] = str.strip(commit.message) commit_collection["commit_time"] = str( commit.committed_datetime) commit_collection["committer_name"] = commit.author.name commit_collection["committer_email"] = commit.author.email diff_files = [] if not commit.parents: continue else: for diff in commit.parents[0].diff(commit): diff_file = {} diff_file["file_path"] = diff.a_path diff_file["change_type"] = diff.change_type diff_file["lang"] = os.path.splitext( diff.a_path)[1][1:] diff_files.append(diff_file) commit_collection["diff_files"] = diff_files commit_collection["parent_commit_num"] = len(commit.parents) commits_list.append(commit_collection) repo_dump = json.dumps(commits_list, indent=2, ensure_ascii=False) with open(os.path.join(stats_dir, repo_name + ".json"), "w") as commit_fd: commit_fd.write(repo_dump) print(repo_name + " done")
def _set_up_repo_state(local_base, git_remote, org, git_local, repo_name, branch, commit, post_clone): git_base = jpth(git_remote, org, repo_name) if not os.path.exists(local_base): repo = Repo.clone_from(git_base + '.git', local_base) post_clone() # FIXME if this does not complete we need to warn or something, it causes errors else: repo = Repo(local_base) nob = repo.active_branch try: nab = getBranch(repo, branch) nab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) nab = repo.active_branch repo.remote().pull() # make sure we are up to date if commit != 'HEAD': repo.git.checkout(commit) return repo, nob
class GitRepository(object): def __init__(self, local_path, repo_url, branch='master'): self.local_path = local_path self.repo_url = repo_url self.repo = None self.initial(branch) def initial(self, branch): if not os.path.exists(self.local_path): os.makedirs(self.local_path) git_local_path = os.path.join(self.local_path, '.git') if not is_git_dir(git_local_path): self.repo = Repo.clone_from(self.repo_url, to_path=self.local_path, branch=branch) else: self.repo = Repo(self.local_path) def pull(self): self.repo.git.pull() def branches(self): branches = self.repo.remote().refs return [ item.remote_head for item in branches if item.remote_head not in [ 'HEAD', ] ] def commits(self): commit_log = self.repo.git.log( '--pretty={"commit":"%h","author":"%an","summary":"%s","date":"%cd"}', max_count=50, date='format:%Y-%m-%d %H:%M') return [json.loads(item) for item in commit_log.split('\n')] def tags(self): return [tag.name for tag in self.repo.tags] def change_to_branch(self, branch): self.repo.git.checkout(branch) def change_to_commit(self, branch, commit): self.change_to_branch(branch=branch) self.repo.git.reset('--hard', commit) def change_to_tag(self, tag): self.repo.git.checkout(tag) pass
def scigraph_build(zip_location, git_remote, org, git_local, branch, commit, clean=False, check_built=False, cleanup_later=False, quiet=False): COMMIT_LOG = 'last-built-commit.log' repo_name = 'SciGraph' remote = jpth(git_remote, org, repo_name) local = jpth(git_local, repo_name) commit_log_path = jpth(local, COMMIT_LOG) if not os.path.exists(local): repo = Repo.clone_from(remote + '.git', local) elif not Path(local, '.git').exists(): repo = Repo.clone_from(remote + '.git', local) else: repo = Repo(local) if not os.path.exists(commit_log_path): last_commit = None else: with open(commit_log_path, 'rt') as f: last_commit = f.read().strip() sob = repo.active_branch try: sab = getBranch(repo, branch) sab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) sab = repo.active_branch repo.remote().pull() if commit != 'HEAD': repo.git.checkout(commit) scigraph_commit = repo.head.object.hexsha scigraph_commit_short = scigraph_commit[:COMMIT_HASH_HEAD_LEN] bin_location = zip_location / 'bin' os.environ['PATH'] = bin_location.as_posix() + ':' + os.environ.get( 'PATH', '') if not bin_location.exists(): bin_location.mkdir() # hack to make the scigraph-load we are about to create available as a command # so that it matches the usual scigraph-load behavior def zip_name(wild=False): return (repo_name + '-' + branch + '-services' + '-' + ('*' if wild else TODAY()) + '-' + scigraph_commit_short + '.zip') def reset_state(original_branch=sob): original_branch.checkout() with execute_regardless( reset_state, only_exception=cleanup_later ): # FIXME this fails when we need to load the graph if we start on master :/ # main if scigraph_commit != last_commit or clean: print('SciGraph not built at commit', commit, 'last built at', last_commit) quiet = '--quiet ' if quiet else '' build_command = ( 'cd ' + local + f';export HASH={scigraph_commit_short}' ';sed -i "/<name>SciGraph<\/name>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" pom.xml' ';sed -i "/<artifactId>scigraph<\/artifactId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-analysis/pom.xml' ';sed -i "/<groupId>io.scigraph<\/groupId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-core/pom.xml' ';sed -i "/<artifactId>scigraph<\/artifactId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-entity/pom.xml' ';sed -i "/<groupId>io.scigraph<\/groupId>/{N;s/<version>.\+<\/version>/<version>${HASH}<\/version>/}" SciGraph-services/pom.xml' f'; mvn {quiet}clean -DskipTests -DskipITs install' '; cd SciGraph-services' f'; mvn {quiet}-DskipTests -DskipITs package') if check_built: print('SciGraph has not been built.') raise NotBuiltError('SciGraph has not been built.') out = os.system(build_command) print(out) if out: scigraph_commit = 'FAILURE' with open(commit_log_path, 'wt') as f: f.write(scigraph_commit) else: print('SciGraph already built at commit', scigraph_commit) wildcard = jpth(zip_location, zip_name(wild=True)) try: services_zip = glob(wildcard)[ 0] # this will error if the zip was moved return scigraph_commit, services_zip, reset_state except IndexError: pass # we need to copy the zip out again # services zip zip_filename = f'scigraph-services-{scigraph_commit_short}.zip' services_zip_temp = Path(local, 'SciGraph-services', 'target', zip_filename) services_zip = jpth(zip_location, zip_name()) shutil.copy(services_zip_temp, services_zip) core_jar = Path( local, 'SciGraph-core', 'target', f'scigraph-core-{scigraph_commit_short}-jar-with-dependencies.jar') scigraph_load = f'''#!/usr/bin/env sh /usr/bin/java \\ -cp "{core_jar.as_posix()}" \\ io.scigraph.owlapi.loader.BatchOwlLoader $@''' slf = bin_location / 'scigraph-load' with open(slf, 'wt') as f: f.write(scigraph_load) os.chmod(slf, 0o0755) return scigraph_commit, services_zip, reset_state
def scigraph_build(zip_location, git_remote, org, git_local, branch, commit, clean=False, check_built=False, cleanup_later=False): COMMIT_LOG = 'last-built-commit.log' repo_name = 'SciGraph' remote = jpth(git_remote, org, repo_name) local = jpth(git_local, repo_name) commit_log_path = jpth(local, COMMIT_LOG) load_base = ('cd {}; '.format(jpth(local, 'SciGraph-core')) + 'mvn exec:java ' '-Dexec.mainClass="io.scigraph.owlapi.loader.BatchOwlLoader" ' '-Dexec.args="-c {config_path}"') if not os.path.exists(local): repo = Repo.clone_from(remote + '.git', local) else: repo = Repo(local) if not os.path.exists(commit_log_path): last_commit = None else: with open(commit_log_path, 'rt') as f: last_commit = f.read().strip() sob = repo.active_branch try: sab = getBranch(repo, branch) sab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) sab = repo.active_branch repo.remote().pull() if commit != 'HEAD': repo.git.checkout(commit) scigraph_commit = repo.head.object.hexsha def zip_name(wild=False): return (repo_name + '-' + branch + '-services' + '-' + ('*' if wild else TODAY()) + '-' + scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '.zip') def reset_state(original_branch=sob): original_branch.checkout() with execute_regardless( reset_state, only_exception=cleanup_later ): # FIXME this fails when we need to load the graph if we start on master :/ # main if scigraph_commit != last_commit or clean: print('SciGraph not built at commit', commit, 'last built at', last_commit) build_command = ('cd ' + local + '; mvn clean -DskipTests -DskipITs install' '; cd SciGraph-services' '; mvn -DskipTests -DskipITs package') if check_built: print('SciGraph has not been built.') raise NotBuiltError('SciGraph has not been built.') out = os.system(build_command) print(out) if out: scigraph_commit = 'FAILURE' with open(commit_log_path, 'wt') as f: f.write(scigraph_commit) else: print('SciGraph already built at commit', scigraph_commit) wildcard = jpth(zip_location, zip_name(wild=True)) try: services_zip = glob(wildcard)[ 0] # this will error if the zip was moved return scigraph_commit, load_base, services_zip, reset_state except IndexError: pass # we need to copy the zip out again # services zip zip_filename = 'scigraph-services-*-SNAPSHOT.zip' services_zip_temp = glob( jpth(local, 'SciGraph-services', 'target', zip_filename))[0] services_zip = jpth(zip_location, zip_name()) shutil.copy(services_zip_temp, services_zip) return scigraph_commit, load_base, services_zip, reset_state
def __init__(self, zip_location, git_remote, org, git_local, repo_name, branch, commit, remote_base, load_base, graphload_config, patch_config, patch, scigraph_commit, post_clone=lambda: None, check_built=False): local_base = jpth(git_local, repo_name) git_base = jpth(git_remote, org, repo_name) if not os.path.exists(local_base): repo = Repo.clone_from(git_base + '.git', local_base) post_clone( ) # FIXME if this does not complete we need to warn or something, it causes errors else: repo = Repo(local_base) nob = repo.active_branch try: nab = getBranch(repo, branch) nab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) nab = repo.active_branch repo.remote().pull() # make sure we are up to date if commit != 'HEAD': repo.git.checkout(commit) # TODO consider dumping metadata in a file in the folder too? def folder_name(scigraph_commit, wild=False): ontology_commit = repo.head.object.hexsha[:COMMIT_HASH_HEAD_LEN] return (repo_name + '-' + branch + '-graph' + '-' + ('*' if wild else TODAY()) + '-' + scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '-' + ontology_commit) def make_folder_zip(wild=False): folder = folder_name(scigraph_commit, wild) graph_path = jpth(zip_location, folder) zip_path = graph_path + '.zip' if wild: return graph_path, zip_path zip_name = os.path.basename(zip_path) zip_dir = os.path.dirname(zip_path) zip_command = ' '.join( ('cd', zip_dir, ';', 'zip -r', zip_name, folder)) return graph_path, zip_path, zip_command graph_path, zip_path, zip_command = make_folder_zip() wild_graph_path, wild_zip_path = make_folder_zip(wild=True) (config, config_path, ontologies) = self.make_graphload_config(graphload_config, graph_path, remote_base, local_base, zip_location) load_command = load_base.format( config_path=config_path) # 'exit 1' to test print(load_command) # replace raw github imports with ontology.neuinfor iris to simplify import chain fix_imports = "find " + local_base + " -name '*.ttl' -exec sed -i 's/<http.\+\/ttl\//<http:\/\/ontology.neuinfo.org\/NIF\/ttl\//' {} \;" os.system(fix_imports) def reset_state(original_branch=nob): repo.git.checkout('--', local_base) original_branch.checkout() with execute_regardless( reset_state ): # FIXME start this immediately after we obtain nob? # main if patch: # FIXME TODO XXX does scigraph load from the catalog!??!?? # because it seems like doid loads correctly without using local_versions # which would be cool, if confusing local_versions = tuple(do_patch(patch_config, local_base)) else: local_versions = tuple() itrips = local_imports( remote_base, local_base, ontologies, local_versions=local_versions, dobig=True) # SciGraph doesn't support catalog.xml catalog = make_catalog(itrips) with open(Path(local_base, 'catalog.xml'), 'wt') as f: f.write(catalog) maybe_zip_path = glob(wild_zip_path) if not maybe_zip_path: if check_built: print('The graph has not been loaded.') raise NotBuiltError('The graph has not been loaded.') failure = os.system(load_command) if failure: if os.path.exists(graph_path): shutil.rmtree(graph_path) else: os.rename( config_path, # save the config for eaiser debugging jpth(graph_path, os.path.basename(config_path))) failure = os.system(zip_command) # graphload zip else: zip_path = maybe_zip_path[0] # this way we get the actual date print('Graph already loaded at', zip_path) # this needs to be run when the branch is checked out # FIXME might be worth adding this to the load config? self.ontologies = [ get_iri(load_header(rec['url'])) for rec in config['ontologies'] ] self.zip_path = zip_path self.itrips = itrips self.config = config
def check(repo_obj): global row global commit_cnt global changed_files_cnt global all_commit_cnt global all_changed_cnt repo_name = repo_obj["owner"]["login"] + "-" + repo_obj["name"] # print("="*8, "start to check repo ", repo_name, "="*8) commit_total = 0 commit_cross = 0 percent = 0 shas = set() repo = Repo(os.path.join(work_dir, repo_name)) for b in repo.remote().fetch(): if '/' not in b.name: continue print("start to check branch {} of {}".format(b.name, repo_name)) branch_name = b.name.split('/')[1] repo.git.checkout('-B', branch_name, b.name) commits = list(repo.iter_commits()) for idx, commit in enumerate(commits): if str(commit) in shas: continue else: shas.add(str(commit)) if len(commit.parents) > 1: continue commit_total += 1 xml_cnt = 0 kot_jav_cnt = 0 all_commit_cnt += 1 # if idx == len(commits)-1: file_list = list(commit.stats.files) all_changed_cnt += len(file_list) for file in file_list: if len(file) > 4 and file[-4:] == '.xml': xml_cnt += 1 elif len(file) > 3 and file[-3:] == '.kt' or len( file) > 5 and file[-5:] == '.java': kot_jav_cnt += 1 if xml_cnt >= 1 and kot_jav_cnt >= 1: commit_cnt += 1 changed_files_cnt += len(file_list) commit_cross += 1 # break # diff_index = commit.diff(commits[idx+1]) # for diff_item in diff_index: # if diff_item.a_path[-4:] == '.xml': # xml_cnt += 1 # elif diff_item.a_path[-3:] == '.kt' or diff_item.a_path[-5:] == '.java': # kot_jav_cnt += 1 # if xml_cnt >= 1 and kot_jav_cnt >= 1: # commit_cross += 1 percent = float(commit_cross) / commit_total repo_name_full = repo_obj["full_name"] res = "{} Total: {}, Cross: {}, Percent: {}".format( repo_name_full, commit_total, commit_cross, percent) print(res) percent_coll.append((repo_obj["full_name"], percent * 100)) # print("="*8, "check repo ", repo_name, "completed", "="*8) return commit_total, commit_cross, percent
class GitRepository(object): """ git仓库管理 """ def __init__(self, local_path, repo_url, branch='master'): self.local_path = local_path self.repo_url = repo_url self.repo = None self.initial(repo_url, branch) def initial(self, repo_url, branch): """ 初始化git仓库 :param repo_url: :param branch: :return: """ if not os.path.exists(self.local_path): os.makedirs(self.local_path) git_local_path = os.path.join(self.local_path, '.git') if not is_git_dir(git_local_path): self.repo = Repo.clone_from(repo_url, to_path=self.local_path, branch=branch) else: self.repo = Repo(self.local_path) def pull(self): """ 从线上拉最新代码 :return: """ self.repo.git.pull() def branches(self): """ 获取所有分支 :return: """ branches = self.repo.remote().refs return [item.remote_head for item in branches if item.remote_head not in ['HEAD', ]] def commits(self): """ 获取所有提交记录 :return: """ commit_log = self.repo.git.log('--pretty={"commit":"%h","author":"%an","summary":"%s","date":"%cd"}', max_count=50, date='format:%Y-%m-%d %H:%M') log_list = commit_log.split("\n") return [eval(item) for item in log_list] def tags(self): """ 获取所有tag :return: """ return [tag.name for tag in self.repo.tags] def change_to_branch(self, branch): """ 切换分值 :param branch: :return: """ self.repo.git.checkout(branch) def change_to_commit(self, branch, commit): """ 切换commit :param branch: :param commit: :return: """ self.change_to_branch(branch=branch) self.repo.git.reset('--hard', commit) def change_to_tag(self, tag): """ 切换tag :param tag: :return: """ self.repo.git.checkout(tag)
def check(repo_name): # global commit_cnt # global changed_files_cnt # global all_commit_cnt # global all_changed_cnt global multi_lang global other shas = set() repo = Repo(os.path.join(work_dir, repo_name)) for b in repo.remote().fetch(): if '/' not in b.name: continue print("start to check branch {} of {}".format(b.name, repo_name)) # branch_name = b.name.split('/')[1] # repo.git.checkout('-B', branch_name, b.name) commits = list(repo.iter_commits('remotes/' + b.name)) for idx, commit in enumerate(commits): if str(commit) in shas: continue else: shas.add(str(commit)) if len(commits[idx - 1].parents) > 1: continue if idx == 0: continue xml_cnt = 0 kot_jav_cnt = 0 # if idx == len(commits)-1: file_list = list(commits[idx - 1].stats.files) dir_set = set() for file in file_list: file = file.split(' => ')[-1] dir = file.split('/')[0] if dir == file: dir_set.add('.') else: dir_set.add(dir) if len(file) > 4 and file[-4:] == '.xml': xml_cnt += 1 elif len(file) > 3 and file[-3:] == '.kt' or len( file) > 5 and file[-5:] == '.java': kot_jav_cnt += 1 dir_cnt = len(dir_set) patch = repo.git.diff(commit.tree, commits[idx - 1].tree).split('\n') hunks_cnt = 0 added = 0 deleted = 0 for line in patch: if len(line) >= 2 and line[0] == '@' and line[1] == '@': hunks_cnt += 1 elif len(line) >= 1 and line[0] == '+' and (len(line) < 3 or line[1] != '+' or line[2] != '+'): added += 1 elif len(line) >= 1 and line[0] == '-' and (len(line) < 3 or line[1] != '-' or line[2] != '-'): deleted += 1 commit_type = "" if xml_cnt >= 1 and kot_jav_cnt >= 1: commit_type = "Multi-lang" multi_lang += 1 else: commit_type = "Other" other += 1 with open(os.path.join(csv_dir, metric_type[0] + ".csv"), 'a', newline="") as csv_fd: csv_writer = csv.writer(csv_fd, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow([commit_type, len(file_list)]) if len(file_list) == 0: print("!!" + str(commits[idx - 1])) with open(os.path.join(csv_dir, metric_type[1] + ".csv"), 'a', newline="") as csv_fd: csv_writer = csv.writer(csv_fd, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow([commit_type, hunks_cnt]) with open(os.path.join(csv_dir, metric_type[2] + ".csv"), 'a', newline="") as csv_fd: csv_writer = csv.writer(csv_fd, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow([commit_type, added]) with open(os.path.join(csv_dir, metric_type[3] + ".csv"), 'a', newline="") as csv_fd: csv_writer = csv.writer(csv_fd, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow([commit_type, deleted]) with open(os.path.join(csv_dir, metric_type[4] + ".csv"), 'a', newline="") as csv_fd: csv_writer = csv.writer(csv_fd, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) csv_writer.writerow([commit_type, dir_cnt]) # break # diff_index = commit.diff(commits[idx+1]) # for diff_item in diff_index: # if diff_item.a_path[-4:] == '.xml': # xml_cnt += 1 # elif diff_item.a_path[-3:] == '.kt' or diff_item.a_path[-5:] == '.java': # kot_jav_cnt += 1 # if xml_cnt >= 1 and kot_jav_cnt >= 1: # commit_cross += 1 print("repo {} done".format(repo_name))
def getQRCommits(repo_name): global QRCommits_cnt repo_cnt = 0 commits_list = [] shas = set() repo = Repo(os.path.join(work_dir, repo_name)) for b in repo.remote().fetch(): if '/' not in b.name: continue print("start to check branch {} of {}".format(b.name, repo_name)) branch_name = b.name.split('/')[1] repo.git.checkout('-B', branch_name, b.name) commits = list(repo.iter_commits(branch_name)) for idx, commit in enumerate(commits): if str(commit) in shas: continue else: shas.add(str(commit)) commit_collection = {} xml_cnt = 0 kot_jav_cnt = 0 file_list = list(commit.stats.files) for file in file_list: if len(file) > 4 and file[-4:] == '.xml': xml_cnt += 1 elif len(file) > 3 and file[-3:] == '.kt' or len( file) > 5 and file[-5:] == '.java': kot_jav_cnt += 1 if xml_cnt >= 1 and kot_jav_cnt >= 1: if idx != 0: valid = False for f in list(commits[idx - 1].stats.files): if '.xml' in f or '.kt' in f or '.java' in f: valid = True break if not valid: continue time_next_commit = commits[ idx - 1].committed_datetime - commit.committed_datetime if time_next_commit.total_seconds() / 60 <= 30 and len( commit.parents) < 2 and len( commits[idx - 1].parents) < 2: QRCommits_cnt += 1 repo_cnt += 1 commit_collection["commit_id"] = str(commit) commit_collection["commit_time"] = str( commit.committed_datetime) commit_collection["commit_msg"] = str.strip( commit.message) commit_collection["remedy_id"] = str(commits[idx - 1]) commit_collection["remedy_time"] = str( commits[idx - 1].committed_datetime) commit_collection["remedy_msg"] = str.strip( commits[idx - 1].message) commits_list.append(commit_collection) # diff_files = [] # if not commit.parents: # continue # else: # for diff in commit.diff(commit.parents[0]): # diff_file = {} # diff_file["file_path"] = diff.a_path # diff_file["change_type"] = diff.change_type # diff_file["lang"] = os.path.splitext(diff.a_path)[1][1:] # diff_files.append(diff_file) # commit_collection["diff_files"] = diff_files # commit_collection["parent_commit_num"] = len(commit.parents) repo_dump = json.dumps(commits_list, indent=2, ensure_ascii=False) with open(os.path.join(stats_dir, repo_name + "_" + str(repo_cnt)), "w") as commit_fd: commit_fd.write(repo_dump) print(repo_name + " done with {} commits".format(repo_cnt))
class GitRepository(object): """ 实现Git仓库自动管理 """ def __init__(self, local_path, repo_url, branch='main'): self.local_path = local_path self.repo_url = repo_url self.repo = None self.initial(repo_url, branch) # self.commitMessage = commitMessage def initial(self, repo_url, branch): """ 初始化git仓库 """ if not os.path.exists(self.local_path): os.makedirs(self.local_path) git_local_path = os.path.join(self.local_path, '.git') if not is_git_dir(git_local_path): self.repo = Repo.clone_from(repo_url, to_path=self.local_path, branch=branch) else: self.repo = Repo(self.local_path) def pull(self): """ 从线上拉最新代码 :return: """ self.repo.git.pull() def branches(self): """ 获取所有分支 :return: """ branches = self.repo.remote().refs return [ item.remote_head for item in branches if item.remote_head not in [ 'HEAD', ] ] # def commits_log(self): # """ # 获取所有提交记录 # :return: # """ # commit_log = Repo.git.log('--pretty={"commit":"%h","author":"%an","summary":"%s","date":"%cd"}', max_count=50, # date='format:%Y-%m-%d %H:%M') # log_list = commit_log.split("\n") # return [eval(item) for item in log_list] def add(self): #cmd = "git add ." + self.local_path cmd = "git add ." process = subprocess.Popen(cmd, shell=True) process.wait() returnCode = process.returncode if returnCode != 0: print(" add returnCode", returnCode) else: print('add success') self.commit() def push(self): cmd = "git push" process = subprocess.Popen(cmd, shell=True) process.wait() returnCode = process.returncode if returnCode != 0: print("push returnCode", returnCode) else: print('push success') # sendMessage({ # "fileName": "api文档 : \n\n已更新,请注意查看! \n" +"\n更新信息: {}".format( # commitMessage), # "text": time.strftime("%Y/%m/%d %H:%M"), # "error": False # }) # pass def commit(self): msg = input("输入提交commit信息:") commitMessage = time.strftime("%Y/%m/%d %H:%M") + '-' + msg cmd = "git commit -m '{}'".format(commitMessage) process = subprocess.Popen(cmd, shell=True) process.wait() self.push() def tags(self): """ 获取所有tag :return: """ return [tag.name for tag in self.repo.tags] def change_to_branch(self, branch): """ 切换分支 :param branch: :return: """ self.repo.git.checkout(branch) def change_to_commit(self, branch, commit): """ 切换commit :param branch: :param commit: :return: """ self.change_to_branch(branch=branch) self.repo.git.reset('--hard', commit) def change_to_tag(self, tag): """ 切换tag :param tag: :return: """ self.repo.git.checkout(tag)
def computeRepo(repo_name): global issue_cnt global chinese_mst_cnt shas = set() print(8 * '=', "start to handle repo ", repo_name, 8 * '=') repo = Repo(os.path.join(repo_dir, repo_name)) for b in repo.remote().fetch(): if '/' not in b.name: continue print("start to check branch {} of {}".format(b.name, repo_name)) branch_name = b.name.split('/')[1] repo.git.checkout('-B', branch_name, b.name) commits = list(repo.iter_commits(branch_name)) for idx, commit in enumerate(commits): if str(commit) in shas: continue else: shas.add(str(commit)) xml_cnt = 0 kot_jav_cnt = 0 for file in list(commit.stats.files): if len(file) > 4 and file[-4:] == '.xml': xml_cnt += 1 elif len(file) > 3 and file[-3:] == '.kt' or len( file) > 5 and file[-5:] == '.java': kot_jav_cnt += 1 if xml_cnt >= 1 and kot_jav_cnt >= 1: msg = commit.message # msg = str.strip( # translator.translate(msg, # lang_tgt='en')).lower() # print(msg) if reg_issue.search(msg) and len(commit.parents) < 2: msg = "fix " + msg issue_cnt += 1 if (check_contain_chinese(msg)): chinese_mst_cnt += 1 seg_list = jieba.cut(msg, cut_all=False) cn = True else: seg_list = wordninja.split(msg.lower()) cn = False for seg in seg_list: if cn is True and check_contain_chinese(seg) is False: seg_split_again = wordninja.split(seg) for _seg in seg_split_again: if len( _seg ) > 1 and _seg != '\t' and _seg != '\n' and _seg != '\r\n' and _seg not in stop_words: if _seg in word_bags: word_bags[_seg] += 1 else: word_bags[_seg] = 1 elif cn is True: if len( seg ) > 1 and seg != '\t' and seg != '\n' and seg != '\r\n' and seg not in stop_words: if seg in word_bags: word_bags[seg] += 1 else: word_bags[seg] = 1 else: if len( seg ) > 1 and seg != '\t' and seg != '\n' and seg != '\r\n' and seg not in stop_words: if seg in word_bags: word_bags[seg] += 1 else: word_bags[seg] = 1 print(8 * '=', "repo ", repo_name, "done", 8 * '=')
def repro_loader(zip_location, git_remote, org, git_local, repo_name, branch, commit, remote_base, load_base, graphload_config, scigraph_commit, post_clone=lambda: None, check_built=False): local_base = jpth(git_local, repo_name) git_base = jpth(git_remote, org, repo_name) if not os.path.exists(local_base): repo = Repo.clone_from(git_base + '.git', local_base) post_clone( ) # FIXME if this does not complete we need to warn or something, it causes errors else: repo = Repo(local_base) nob = repo.active_branch try: nab = getBranch(repo, branch) nab.checkout() except ValueError: # usually indicates a remote branch repo.git.checkout(branch) nab = repo.active_branch repo.remote().pull() # make sure we are up to date if commit != 'HEAD': repo.git.checkout(commit) # TODO consider dumping metadata in a file in the folder too? def folder_name(scigraph_commit, wild=False): ontology_commit = repo.head.object.hexsha[:COMMIT_HASH_HEAD_LEN] return (repo_name + '-' + branch + '-graph' + '-' + ('*' if wild else TODAY) + '-' + scigraph_commit[:COMMIT_HASH_HEAD_LEN] + '-' + ontology_commit) def make_folder_zip(wild=False): folder = folder_name(scigraph_commit, wild) graph_path = jpth(zip_location, folder) zip_path = graph_path + '.zip' if wild: return graph_path, zip_path zip_name = os.path.basename(zip_path) zip_dir = os.path.dirname(zip_path) zip_command = ' '.join( ('cd', zip_dir, ';', 'zip -r', zip_name, folder)) return graph_path, zip_path, zip_command graph_path, zip_path, zip_command = make_folder_zip() wild_graph_path, wild_zip_path = make_folder_zip(wild=True) # config graphload.yaml from template graphload_config_template = graphload_config + '.template' with open(graphload_config_template, 'rt') as f: config = yaml.load(f) config['graphConfiguration']['location'] = graph_path config['ontologies'] = [{ k: v.replace(remote_base, local_base) if k == 'url' else v for k, v in ont.items() } for ont in config['ontologies']] config_path = jpth(zip_location, 'graphload-' + TODAY + '.yaml') with open(config_path, 'wt') as f: yaml.dump(config, f, default_flow_style=False) ontologies = [ont['url'] for ont in config['ontologies']] load_command = load_base.format(config_path=config_path) print(load_command) def reset_state(original_branch=nob): original_branch.checkout() # return to original state (reset --hard) repo.head.reset( index=True, working_tree=True ) # FIXME we need to not run anything if there are files added to staging with execute_regardless( reset_state): # FIXME start this immediately after we obtain nob? # main itrips = local_imports( remote_base, local_base, ontologies) # SciGraph doesn't support catalog.xml maybe_zip_path = glob(wild_zip_path) if not maybe_zip_path: if check_built: print('The graph has not been loaded.') raise NotBuiltError('The graph has not been loaded.') failure = os.system(load_command) if failure: if os.path.exists(graph_path): shutil.rmtree(graph_path) else: os.rename( config_path, # save the config for eaiser debugging jpth(graph_path, os.path.basename(config_path))) failure = os.system(zip_command) # graphload zip else: zip_path = maybe_zip_path[0] # this way we get the actual date print('Graph already loaded at', zip_path) return zip_path, itrips