def update_baseline(c):
c.execute("select sha from commits where in_baseline is 0")
for (sha) in c.fetchall():
try:
subprocess.check_output(
['git', 'merge-base', '--is-ancestor', sha,
rebase_baseline()])
c.execute("UPDATE commits SET in_baseline=1 where sha='%s'" % sha)
except:
pass
def update_upstreamdb():
start = rebase_baseline()
try:
# see if we previously handled anything. If yes, use it.
# Otherwise re-create database.
conn = sqlite3.connect(upstreamdb)
c = conn.cursor()
c.execute("select sha from tip")
sha = c.fetchone()
conn.close()
if sha and sha[0] != "":
start = sha[0]
# In this case, the baseline may have changed. Assume that it only
# moves forward. Update database to reflect new baseline.
update_baseline(c, start, rebase_baseline())
else:
fail
except:
createdb(upstreamdb, mktables)
os.chdir(upstream_path)
upstream_drop = start + '..' + rebase_target_tag()
upstream_pick = rebase_target_tag() + '..'
conn = sqlite3.connect(upstreamdb)
c = conn.cursor()
handle(c, upstream_drop, 1)
handle(c, upstream_pick, 0)
conn.commit()
conn.close()
os.chdir(workdir)
def init_spreadsheet(sheet):
try:
with open(rebase_filename, 'r') as file:
id = file.read()
request = sheet.get(spreadsheetId=id, ranges = [ ], includeGridData=False)
response = request.execute()
sheets = response.get('sheets')
delete_sheets(sheet, id, sheets)
except:
id = create_spreadsheet(sheet, 'Rebase %s -> %s' %
(rebase_baseline(), rebase_target_tag()))
with open(rebase_filename, 'w') as file:
file.write(id)
return id
def init_spreadsheet(sheet):
try:
with open(stats_filename, 'r') as file:
id = file.read().strip('\n')
request = sheet.get(spreadsheetId=id, ranges=[], includeGridData=False)
response = request.execute()
sheets = response.get('sheets')
delete_sheets(sheet, id, sheets)
except:
id = create_spreadsheet(
sheet,
'Backlog Status for chromeos-%s' % rebase_baseline().strip('v'))
with open(stats_filename, 'w') as file:
file.write(id)
return id
def main():
sheet = genlib.init_spreadsheet(stats_filename,
'Backlog Status for chromeos-%s' % rebase_baseline().strip('v'))
summary_rows = create_summary(sheet)
topic_stats_sheet, topic_stats_rows, topic_stats_columns = create_topic_stats(sheet)
add_backlog_chart(sheet, summary_rows)
add_age_chart(sheet, summary_rows)
add_stats_chart(sheet, topic_stats_sheet, topic_stats_rows, topic_stats_columns)
genlib.move_sheet(sheet, 0, 4)
genlib.hide_sheet(sheet, 0, True)
genlib.move_sheet(sheet, topic_stats_sheet, 5)
genlib.hide_sheet(sheet, topic_stats_sheet, True)
def main():
sheet = genlib.init_spreadsheet(
rebase_filename,
'Rebase %s -> %s' % (rebase_baseline(), rebase_target_tag()))
get_other_topic_id()
requests = []
create_summary(requests)
add_topics_sheets(requests)
genlib.doit(sheet, requests)
requests = []
add_commits(requests)
# Now auto-resize columns A, B, C, and G in Summary sheet
genlib.resize_sheet(requests, 0, 0, 3)
genlib.resize_sheet(requests, 0, 6, 7)
# Add description after resizing
add_description(requests)
genlib.doit(sheet, requests)
def update_upstreamdb():
start = rebase_baseline()
try:
# see if we previously handled anything. If yes, use it.
# Otherwise re-create database.
conn = sqlite3.connect(upstreamdb)
c = conn.cursor()
c.execute("select sha from tip")
sha, = c.fetchone()
conn.close()
print("Last handled sha: %s" % sha)
if sha:
start = sha
# In this case, integration status may have changed.
# Update database accordingly.
update_baseline()
else:
fail
except:
createdb(upstreamdb, mktables)
os.chdir(upstream_path)
print("Starting with SHA %s" % start)
range = start + '..'
conn = sqlite3.connect(upstreamdb)
c = conn.cursor()
# init_tags(c)
# conn.commit()
update_patchids(c)
conn.commit()
handle(c, range)
conn.commit()
conn.close()
os.chdir(workdir)
def update_commits():
'''
Get complete list of commits from rebase baseline.
Assume that the baseline branch exists and has been checked out.
'''
conn = sqlite3.connect(rebasedb)
c = conn.cursor()
os.chdir(chromeos_path)
commits = subprocess.check_output(['git', 'log', '--no-merges', '--abbrev=12', '--reverse',
'--format=%at%x01%ct%x01%h%x01%an%x01%ae%x01%s',
rebase_baseline() + '..'])
prevdate = 0
mprevdate = 0
for commit in commits.splitlines():
if commit != "":
elem = commit.split("\001", 5)
authored=elem[0]
committed=elem[1]
sha = elem[2]
contact = elem[3]
email = elem[4]
if '@google.com' not in email and '@chromium.org' not in email and '@collabora.com' not in email:
ncontact, nemail = get_contact(sha)
if ncontact:
contact = ncontact
email = nemail
contact = contact.decode('latin-1') \
if isinstance(contact, str) else contact
email = email.decode('latin-1') \
if isinstance(email, str) else email
subject = elem[5].rstrip('\n')
subject = subject.decode('latin-1') \
if isinstance(subject, str) else subject
ps = subprocess.Popen(['git', 'show', sha], stdout=subprocess.PIPE)
spid = subprocess.check_output(['git', 'patch-id'], stdin=ps.stdout)
patchid = spid.split(" ", 1)[0]
# Make sure date is unique and in ascending order.
date = int(committed)
if date == prevdate:
date = mprevdate + 1
else:
prevdate = date
date = date * 1000
mprevdate = date
# Do nothing if the sha is already in the commit table.
c.execute("select sha from commits where sha='%s'" % sha)
found = c.fetchone()
if found:
continue
# check for cherry pick lines. If so, record the upstream SHA associated
# with this commit. Only look for commits which may be upstream or may
# have been merged from a stable release.
usha=""
if not chromium.match(subject):
u = upstream.match(subject)
desc = subprocess.check_output(['git', 'show', '-s', sha])
for d in desc.splitlines():
m=None
if u:
m = cherrypick.search(d)
else:
m = stable.search(d)
if not m:
m = stable2.search(d)
if m:
usha=m.group(2)[:12]
# The patch may have been picked multiple times; only record
# the first entry.
break
# Search for embedded Change-Id string.
# If found, add it to database.
desc = subprocess.check_output(['git', 'show', '-s', sha])
for d in desc.decode('latin-1').splitlines():
chid = changeid.match(d)
if chid:
chid = chid.group(1)
break
# Initially assume we'll drop everything because it is not listed when
# running "rebase -i". Before doing that, check if the commit is a
# stable release commit. If so, mark it accordingly.
reason = 'upstream'
c.execute("select sha from stable where sha is '%s'" % sha)
if c.fetchone():
reason = 'stable'
c.execute("INSERT INTO commits(date, created, updated, authored, committed, contact, email, sha, usha, patchid, changeid, subject, disposition, reason) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(date, NOW(), NOW(), authored, committed, contact, email, sha, usha, patchid, chid, subject, "drop", reason,))
filenames = subprocess.check_output(['git', 'show', '--name-only',
'--format=', sha])
for fn in filenames.splitlines():
if fn != "":
c.execute("INSERT INTO files(sha, filename) VALUES (?, ?)", (sha, fn,))
conn.commit()
# "git cherry -v <target>" on branch rebase_baseline gives us a list
# of patches to apply.
patches = subprocess.check_output(['git', 'cherry', '-v', rebase_target_tag()])
for patch in patches.splitlines():
elem = patch.split(" ", 2)
# print "patch: " + patch
# print "elem[0]: '%s' elem[1]: '%s' elem[2]: '%s'" % (elem[0], elem[1], elem[2])
if elem[0] == "+":
# patch not found upstream
sha = elem[1][:12]
# Try to find patch in stable branch. If it is there, drop it after all.
# If not, we may need to apply it.
c.execute("select sha, origin from stable where sha is '%s'" % sha)
found = c.fetchone()
if found:
c.execute("UPDATE commits SET disposition=('drop') where sha='%s'" % sha)
c.execute("UPDATE commits SET reason=('%s') where sha='%s'" %
(found[1], sha))
c.execute("UPDATE commits SET updated=('%d') where sha='%s'" % (NOW(), sha))
else:
# We need to check if the commit is already marked as drop
# with a reason other than "upstream". If so, don't update it.
c.execute("select disposition, reason from commits where sha='%s'" % sha)
found = c.fetchone()
if found and found[0] == "drop" and found[1] == "upstream":
c.execute("UPDATE commits SET disposition=('pick') where sha='%s'" % sha)
c.execute("UPDATE commits SET reason=('') where sha='%s'" % sha)
c.execute("UPDATE commits SET updated=('%d') where sha='%s'" % (NOW(), sha))
os.chdir(workdir)
conn.commit()
conn.close()
# -*- coding: utf-8 -*-"
import sqlite3
import os
import re
import subprocess
import time
from config import rebasedb, \
stable_path, android_path, chromeos_path
from common import workdir, stable_baseline, rebase_baseline, rebase_target_tag
stable_commits = rebase_baseline() + '..' + stable_baseline()
baseline_commits = rebase_baseline() + '..'
# "commit" is sometimes seen multiple times, such as with commit 6093aabdd0ee
cherrypick=re.compile("cherry picked from (commit )+([a-z0-9]+)")
stable=re.compile("(commit )+([a-z0-9]+) upstream")
stable2=re.compile("Upstream (commit )+([a-z0-9]+)")
upstream=re.compile("(ANDROID: *|UPSTREAM: *|FROMGIT: *|BACKPORT: *)+(.*)")
chromium=re.compile("(CHROMIUM: *|FROMLIST: *)+(.*)")
changeid=re.compile("\s*Change-Id:\s+(I[a-z0-9]+)")
def NOW():
return int(time.time())
def removedb():
'''
remove database if it exists
'''
try:
os.remove(rebasedb)