def collect_push_list(project): dal=dbaccess.DBAccess() server_list=Server.get_server_list(project.project_server_ip) is_have_new_push=0 if len(server_list)==1: server=server_list[0] conn=dal.build_connection(server) sql_max_time="select 1,TO_CHAR(MAX(git_push_date),'YYYY-MM-DD HH24:MI:SS.US') FROM GIT_PUSH WHERE git_push_project_id='"+str(project.project_id)+"'" max_time=dal.execute_read_sql(sql_max_time) str_filter="" sql="" if server.server_type=="gitlab": items=project.project_path.split("/") if len(items)==2: str_filter+=" and tn.path='"+items[0]+"' and tp.path='"+items[1]+"'" if len(max_time)>0 and max_time[0][1] !=None: str_filter+=" and te.created_at > '"+str(max_time[0][1])+"'" sql="""select tn.path||'/'||tp.path as path,tu.username,tu.name,tu.email,te.created_at,te.data from events te left join projects tp on tp.id=te.project_id left join users tu on tu.id=te.author_id left join namespaces tn on tp.namespace_id=tn.id where te.action=5 str_filter;""" elif server.server_type=="gerrit": if len(max_time)>0 and max_time[0][1] !=None: str_filter+=" and to_char(tp.created_on-interval '8' hour,'yyyy-mm-dd hh24:mi:ss.ff6') > '"+str(max_time[0][1])+"'" str_filter+=" and tc.dest_project_name='"+project.project_path+"'" sql="""select tc.dest_project_name,tae.external_id,ta.full_name,ta.preferred_email,to_char(tp.created_on-interval '8' hour,'yyyy-mm-dd hh24:mi:ss.ff6') as created_on ,tc.change_key,tp.revision from patch_sets tp left join changes tc on tc.change_id=tp.change_id left join accounts ta on ta.account_id=tp.uploader_account_id left join account_external_ids tae on ta.account_id=tae.account_id and tae.external_id like 'gerrit%' where tae.email_address is not null str_filter """ sql=sql.replace("str_filter",str_filter) data_list=dal.execute_read_sql_from_servers(conn,sql) if len(data_list)>0: push_list=[] for item in data_list: push=Push() push.push_project_id=project.project_id push.push_author_username=item[1].replace("gerrit:","") push.push_author_name=item[2] push.push_author_email=item[3] push.push_date=item[4] if server.server_type=="gitlab": items=item[5].split("\n") for item in items: # do not loop useless rows if push.push_type!="" and push.push_revision_before!="" and push.push_revision_after!="": break fields=item.split(":") hash_id_value="" if len(fields)==3 and len(fields[2])>=40: #replace first resivision's double quotation mark if "'" in fields[2]: hash_id_value=fields[2].replace("'","").strip() else: hash_id_value=fields[2].strip() if re.search(":before", item) != None: push.push_revision_before=hash_id_value elif re.search(":after", item) != None: push.push_revision_after=hash_id_value elif re.search(":ref", item) != None: fields=item.split(":") if len(fields)==3 and "refs/tags/" in fields[2]: push.push_type="tag" else: push.push_type="revision" push.push_source="gitlab" elif server.server_type=="gerrit": push.push_revision_before=item[5] push.push_revision_after=item[6] push.push_type="revision" push.push_source="gerrit" push_list.append(push) Push.insert_push_list(push_list) is_have_new_push=1 else: print ('Warning: unexpected exception "%s"' % server_list) return is_have_new_push
def collect(project): #1.Server server_list=Server.get_server_list() #2.Project_Insert new projects new_project_list=Project.get_new_project_list(server_list) if len(new_project_list) >0: Project.insert_new_project_list(new_project_list) #2.Project_Get all projects serial_number=0 #Collect data by server_name server_name="" if len(sys.argv)>1: server_name=sys.argv[1] if server_name=="": project_list=Project.get_project_list() else: project_list=Project.get_project_list(server_name) for project in project_list: serial_number=serial_number+1 print (">>>>>>No%s.Git project url: %s " %(len(project_list)-serial_number, project.project_repository_url)) print (">>>>>>0_Collecting push records") is_have_new_push=Push.collect_push_list(project) if is_have_new_push==0: print (">>>>>>There is nothing new in repository \n") continue # clean workspace git_home=os.getcwd() git_path=git_home+"/"+project.project_name if os.path.isdir(git_path): Util.getpipeoutput(["rm -rf %s " % git_path ]) print (">>>>>>1_Git path: %s" % git_path) print (">>>>>>2_Clone git repository") Util.getpipeoutput(["git clone %s " % project.project_repository_url+project.project_name ]) print (">>>>>>3_Collecting git data") if os.path.isdir(git_path): os.chdir(git_path) #########Begin to collect #Collect new branchs Branch.collect_branch_list(project.project_id) #Query all branchs from database all_branch_list=Branch.get_branch_list(project.project_id) branch_list=[] for branch in all_branch_list: revision_list=[] print(" >>>>>>>>Branch Name:"+branch.branch_name) current_branch=Util.getpipeoutput(["git rev-parse --abbrev-ref HEAD"]) if current_branch!=branch.branch_name: Util.getpipeoutput(["git checkout %s" % branch.branch_name]) # if last_commit_id is empty ,it means that it's a new branch latest_commit_id=Util.getpipeoutput(["git rev-parse HEAD"]) if branch.branch_last_commit_id!=latest_commit_id: #Collect all the Revisions(all commits) branch_total_line=Revision.collect_revision_list(branch,latest_commit_id) #Collect all the files local_file_change_list=File.collect_file_list(branch,latest_commit_id) #Collect all the link Revision_File_Link.collect_link_list(local_file_change_list,branch,latest_commit_id) #Update branch info branch.branch_type="update" branch.branch_total_line=branch_total_line branch.branch_last_commit_id=latest_commit_id branch.branch_contributor_counts = int(Util.getpipeoutput(["git shortlog -s %s" % Util.getlogrange(), "wc -l"])) branch.branch_file_counts=int(Util.getpipeoutput(["git ls-files | wc -l"])) branch_list.append(branch) Branch.update_branch_list(branch_list) Tag.collect_tag_list(project.project_id) # Merge Request # Project LOC #########End os.chdir(git_home) print (">>>>>>4.Delete the git repository diretory\n") if os.path.isdir(git_path): Util.getpipeoutput(["rm -rf %s " % git_path ])