def collect_push_list(project):
dal=dbaccess.DBAccess()
server_list=Server.get_server_list(project.project_server_ip)
is_have_new_push=0
if len(server_list)==1:
server=server_list[0]
conn=dal.build_connection(server)
sql_max_time="select 1,TO_CHAR(MAX(git_push_date),'YYYY-MM-DD HH24:MI:SS.US') FROM GIT_PUSH WHERE git_push_project_id='"+str(project.project_id)+"'"
max_time=dal.execute_read_sql(sql_max_time)
str_filter=""
sql=""
if server.server_type=="gitlab":
items=project.project_path.split("/")
if len(items)==2:
str_filter+=" and tn.path='"+items[0]+"' and tp.path='"+items[1]+"'"
if len(max_time)>0 and max_time[0][1] !=None:
str_filter+=" and te.created_at > '"+str(max_time[0][1])+"'"
sql="""select tn.path||'/'||tp.path as path,tu.username,tu.name,tu.email,te.created_at,te.data
from events te
left join projects tp on tp.id=te.project_id
left join users tu on tu.id=te.author_id
left join namespaces tn on tp.namespace_id=tn.id
where te.action=5 str_filter;"""
elif server.server_type=="gerrit":
if len(max_time)>0 and max_time[0][1] !=None:
str_filter+=" and to_char(tp.created_on-interval '8' hour,'yyyy-mm-dd hh24:mi:ss.ff6') > '"+str(max_time[0][1])+"'"
str_filter+=" and tc.dest_project_name='"+project.project_path+"'"
sql="""select tc.dest_project_name,tae.external_id,ta.full_name,ta.preferred_email,to_char(tp.created_on-interval '8' hour,'yyyy-mm-dd hh24:mi:ss.ff6') as created_on ,tc.change_key,tp.revision
from patch_sets tp
left join changes tc on tc.change_id=tp.change_id
left join accounts ta on ta.account_id=tp.uploader_account_id
left join account_external_ids tae on ta.account_id=tae.account_id and tae.external_id like 'gerrit%'
where tae.email_address is not null str_filter """
sql=sql.replace("str_filter",str_filter)
data_list=dal.execute_read_sql_from_servers(conn,sql)
if len(data_list)>0:
push_list=[]
for item in data_list:
push=Push()
push.push_project_id=project.project_id
push.push_author_username=item[1].replace("gerrit:","")
push.push_author_name=item[2]
push.push_author_email=item[3]
push.push_date=item[4]
if server.server_type=="gitlab":
items=item[5].split("\n")
for item in items:
# do not loop useless rows
if push.push_type!="" and push.push_revision_before!="" and push.push_revision_after!="":
break
fields=item.split(":")
hash_id_value=""
if len(fields)==3 and len(fields[2])>=40:
#replace first resivision's double quotation mark
if "'" in fields[2]:
hash_id_value=fields[2].replace("'","").strip()
else:
hash_id_value=fields[2].strip()
if re.search(":before", item) != None:
push.push_revision_before=hash_id_value
elif re.search(":after", item) != None:
push.push_revision_after=hash_id_value
elif re.search(":ref", item) != None:
fields=item.split(":")
if len(fields)==3 and "refs/tags/" in fields[2]:
push.push_type="tag"
else:
push.push_type="revision"
push.push_source="gitlab"
elif server.server_type=="gerrit":
push.push_revision_before=item[5]
push.push_revision_after=item[6]
push.push_type="revision"
push.push_source="gerrit"
push_list.append(push)
Push.insert_push_list(push_list)
is_have_new_push=1
else:
print ('Warning: unexpected exception "%s"' % server_list)
return is_have_new_push
def collect(project):
#1.Server
server_list=Server.get_server_list()
#2.Project_Insert new projects
new_project_list=Project.get_new_project_list(server_list)
if len(new_project_list) >0:
Project.insert_new_project_list(new_project_list)
#2.Project_Get all projects
serial_number=0
#Collect data by server_name
server_name=""
if len(sys.argv)>1:
server_name=sys.argv[1]
if server_name=="":
project_list=Project.get_project_list()
else:
project_list=Project.get_project_list(server_name)
for project in project_list:
serial_number=serial_number+1
print (">>>>>>No%s.Git project url: %s " %(len(project_list)-serial_number, project.project_repository_url))
print (">>>>>>0_Collecting push records")
is_have_new_push=Push.collect_push_list(project)
if is_have_new_push==0:
print (">>>>>>There is nothing new in repository \n")
continue
# clean workspace
git_home=os.getcwd()
git_path=git_home+"/"+project.project_name
if os.path.isdir(git_path):
Util.getpipeoutput(["rm -rf %s " % git_path ])
print (">>>>>>1_Git path: %s" % git_path)
print (">>>>>>2_Clone git repository")
Util.getpipeoutput(["git clone %s " % project.project_repository_url+project.project_name ])
print (">>>>>>3_Collecting git data")
if os.path.isdir(git_path):
os.chdir(git_path)
#########Begin to collect
#Collect new branchs
Branch.collect_branch_list(project.project_id)
#Query all branchs from database
all_branch_list=Branch.get_branch_list(project.project_id)
branch_list=[]
for branch in all_branch_list:
revision_list=[]
print(" >>>>>>>>Branch Name:"+branch.branch_name)
current_branch=Util.getpipeoutput(["git rev-parse --abbrev-ref HEAD"])
if current_branch!=branch.branch_name:
Util.getpipeoutput(["git checkout %s" % branch.branch_name])
# if last_commit_id is empty ,it means that it's a new branch
latest_commit_id=Util.getpipeoutput(["git rev-parse HEAD"])
if branch.branch_last_commit_id!=latest_commit_id:
#Collect all the Revisions(all commits)
branch_total_line=Revision.collect_revision_list(branch,latest_commit_id)
#Collect all the files
local_file_change_list=File.collect_file_list(branch,latest_commit_id)
#Collect all the link
Revision_File_Link.collect_link_list(local_file_change_list,branch,latest_commit_id)
#Update branch info
branch.branch_type="update"
branch.branch_total_line=branch_total_line
branch.branch_last_commit_id=latest_commit_id
branch.branch_contributor_counts = int(Util.getpipeoutput(["git shortlog -s %s" % Util.getlogrange(), "wc -l"]))
branch.branch_file_counts=int(Util.getpipeoutput(["git ls-files | wc -l"]))
branch_list.append(branch)
Branch.update_branch_list(branch_list)
Tag.collect_tag_list(project.project_id)
# Merge Request
# Project LOC
#########End
os.chdir(git_home)
print (">>>>>>4.Delete the git repository diretory\n")
if os.path.isdir(git_path):
Util.getpipeoutput(["rm -rf %s " % git_path ])
