def collect_link_list(local_file_change_list,branch,latest_commit_id): revision_file_link_list=[] exist_file_list=File.get_file_list(branch.branch_project_id) exist_link_list=Revision_File_Link.get_link_list(branch.branch_project_id) #To avoid repeating to execute git log --numstat if len(local_file_change_list)==0: if (branch.branch_last_commit_id and not branch.branch_last_commit_id.isspace()): local_file_change_list = Util.getpipeoutput(['git log --numstat %s --pretty=format:"%%at|^%%aN|^%%H|^%%s|^MARK" %s' % (" -m", branch.branch_last_commit_id+".."+latest_commit_id)]).split('\n') else: local_file_change_list = Util.getpipeoutput(['git log --numstat %s --pretty=format:"%%at|^%%aN|^%%H|^%%s|^MARK" %s' % (" -m", Util.getlogrange('HEAD'))]).split('\n') hash_id_value="" commit_note="" for item in local_file_change_list: if len(item) == 0: continue if re.search("MARK", item) != None: change = item.split("|^") if len(change)!=0: try: (stamp,author, hashid,note) = (int(change[0]), change[1],change[2],change[3]) hash_id_value=hashid commit_note=note except ValueError: print ('Warning: unexpected line "%s"' % item) else: print ('Warning: unexpected line "%s"' % item) else: change = item.split() file_id_value="" if len(change) == 3: if "Merge" in commit_note: continue filepaths=change[2].split("/") (inserted, deleted,filename,filepath) = (change[0],change[1],filepaths[len(filepaths)-1],change[2]) for item in exist_file_list: if item.file_path.strip()==change[2].strip(): file_id_value=item.file_id break is_have=0 for item in exist_link_list: if item.link_revision_hash_id==hash_id_value and item.link_file_id==file_id_value: is_have=1 #do not store the same hash id and file id record,the same revision in each branchs only store once break if is_have==0: link=Revision_File_Link() link.link_file_id=file_id_value link.link_revision_hash_id=hash_id_value link.link_file_line_added=inserted link.link_file_line_deleted=deleted link.link_project_id=branch.branch_project_id revision_file_link_list.append(link) else: print('Warning: unexpected line "%s"' % item) Revision_File_Link.insert_new_link_list(revision_file_link_list)
def collect(project): #1.Server server_list=Server.get_server_list() #2.Project_Insert new projects new_project_list=Project.get_new_project_list(server_list) if len(new_project_list) >0: Project.insert_new_project_list(new_project_list) #2.Project_Get all projects serial_number=0 #Collect data by server_name server_name="" if len(sys.argv)>1: server_name=sys.argv[1] if server_name=="": project_list=Project.get_project_list() else: project_list=Project.get_project_list(server_name) for project in project_list: serial_number=serial_number+1 print (">>>>>>No%s.Git project url: %s " %(len(project_list)-serial_number, project.project_repository_url)) print (">>>>>>0_Collecting push records") is_have_new_push=Push.collect_push_list(project) if is_have_new_push==0: print (">>>>>>There is nothing new in repository \n") continue # clean workspace git_home=os.getcwd() git_path=git_home+"/"+project.project_name if os.path.isdir(git_path): Util.getpipeoutput(["rm -rf %s " % git_path ]) print (">>>>>>1_Git path: %s" % git_path) print (">>>>>>2_Clone git repository") Util.getpipeoutput(["git clone %s " % project.project_repository_url+project.project_name ]) print (">>>>>>3_Collecting git data") if os.path.isdir(git_path): os.chdir(git_path) #########Begin to collect #Collect new branchs Branch.collect_branch_list(project.project_id) #Query all branchs from database all_branch_list=Branch.get_branch_list(project.project_id) branch_list=[] for branch in all_branch_list: revision_list=[] print(" >>>>>>>>Branch Name:"+branch.branch_name) current_branch=Util.getpipeoutput(["git rev-parse --abbrev-ref HEAD"]) if current_branch!=branch.branch_name: Util.getpipeoutput(["git checkout %s" % branch.branch_name]) # if last_commit_id is empty ,it means that it's a new branch latest_commit_id=Util.getpipeoutput(["git rev-parse HEAD"]) if branch.branch_last_commit_id!=latest_commit_id: #Collect all the Revisions(all commits) branch_total_line=Revision.collect_revision_list(branch,latest_commit_id) #Collect all the files local_file_change_list=File.collect_file_list(branch,latest_commit_id) #Collect all the link Revision_File_Link.collect_link_list(local_file_change_list,branch,latest_commit_id) #Update branch info branch.branch_type="update" branch.branch_total_line=branch_total_line branch.branch_last_commit_id=latest_commit_id branch.branch_contributor_counts = int(Util.getpipeoutput(["git shortlog -s %s" % Util.getlogrange(), "wc -l"])) branch.branch_file_counts=int(Util.getpipeoutput(["git ls-files | wc -l"])) branch_list.append(branch) Branch.update_branch_list(branch_list) Tag.collect_tag_list(project.project_id) # Merge Request # Project LOC #########End os.chdir(git_home) print (">>>>>>4.Delete the git repository diretory\n") if os.path.isdir(git_path): Util.getpipeoutput(["rm -rf %s " % git_path ])
def collect_revision_list(branch,latest_commit_id): revision_list=[] if (branch.branch_last_commit_id and not branch.branch_last_commit_id.isspace()): local_revision_list = Util.getpipeoutput(['git rev-list --pretty=format:"%%H|^%%at|^%%ai|^%%s|^%%aN|^<%%aE>" %s' % branch.branch_last_commit_id+".."+latest_commit_id, 'grep -v ^commit']).split('\n') local_revision_file_changeset_list = Util.getpipeoutput(['git log --shortstat %s --pretty=format:"%%at|^%%aN|^%%H " %s' % (" -m", branch.branch_last_commit_id+".."+latest_commit_id)]).split('\n') else: local_revision_list = Util.getpipeoutput(['git rev-list --pretty=format:"%%H|^%%at|^%%ai|^%%s|^%%aN|^<%%aE>" %s' % Util.getlogrange('HEAD'), 'grep -v ^commit']).split('\n') local_revision_file_changeset_list = Util.getpipeoutput(['git log --shortstat %s --pretty=format:"%%at|^%%aN|^%%H " %s' % (" -m", Util.getlogrange('HEAD'))]).split('\n') #revision list for item in local_revision_list: if "Merge" in item: continue revision= Revision() parts = item.split('|^', 6) try: stamp = int(parts[1]) except ValueError: stamp = 0 hash_id = parts[0] note = parts[3] author = parts[4] mail = parts[5].lstrip('<').rstrip('>') date = datetime.datetime.fromtimestamp(float(stamp)) revision.revision_project_id=branch.branch_project_id revision.revision_branch_id=branch.branch_id revision.revision_hash_id=hash_id revision.revision_author_username=author revision.revision_author_email=mail revision.revision_date=date revision.revision_commit_note=note revision_list.append(revision) #revision list with file changeset hash_id_value="" for changeset in local_revision_file_changeset_list: if len(changeset) == 0: continue files = 0; inserted = 0; deleted = 0 author = None if re.search('files? changed', changeset) == None: changes = changeset.split("|^") if len(changes)!=0: try: (stamp,author, hashid) = (int(changes[0]), changes[1],changes[2]) hash_id_value=hashid files, inserted, deleted = 0, 0, 0 except ValueError: print ('Warning: unexpected line "%s"' % changeset) else: print ('Warning: unexpected line "%s"' % changeset) else: numbers = Util.getstatsummarycounts(changeset) if len(numbers) == 3: (files, inserted, deleted) = map(lambda el : int(el), numbers) branch.branch_total_line += inserted branch.branch_total_line -= deleted else: print ('Warning: failed to handle line "%s"' % line) (files, inserted, deleted) = (0, 0, 0) for revision in revision_list: if revision.revision_hash_id.strip()==hash_id_value.strip(): revision.revision_line_added+=inserted revision.revision_line_deleted+=deleted revision.revision_file_changed+=files Revision.insert_revision_list(revision_list) return branch.branch_total_line