class Connection_Setup:
def __init__(self,url,username,password):
self.url=url
self.gQ=Gerrit_Queries()
if username=='' and password=='':
self.auth=None
else:
self.username=username
self.password=password
self.auth = HTTPDigestAuth(self.username, self.password)
def setConnection(self):
try:
self.rest = GerritRestAPI(url=self.url,auth=self.auth) #############################################
dummy = self.rest.get(self.gQ.testingQuery())
except:
try:
if self.auth is not None:
self.auth = HTTPBasicAuth(self.username, self.password)
self.rest = GerritRestAPI(url=self.url, auth=self.auth)
dummy = self.rest.get(self.gQ.testingQuery())
except:
sys.exit('Authentication Failed!')
return self.rest
def _main():
myauth = HTTPDigestAuth('account', 'password')
rest = GerritRestAPI(url='http://code...', auth=myauth)
try:
"""
conditions
"""
query = ["status:open"]
query += ["owner:leon5209"]
query += ["age:1week"]
changes = rest.get("/changes/?q=%s" % "%20".join(query))
logging.debug("there are %d changes", len(changes))
"""
abandon
"""
for change in changes:
logging.debug("subject : %s ", change['subject'])
logging.debug("change id : %s ", change['change_id'])
# we don't kill draft
if change['status'] == "DRAFT":
logging.debug("skip, we dont abandon draft")
else:
logging.debug("commit out of date , abandon!")
rest.post("/changes/" + change['change_id'] + "/abandon",
json={
"message":
"This Commit is out of date, auto abandon! "
})
except RequestException as err:
logging.error("Error: %s", str(err))
def get_change(url, repo, branch, change_id):
"""
Fetches a change from upstream repo
:param url: URL of upstream gerrit
:param repo: name of repo
:param branch: branch of repo
:param change_id: SHA change id
:return: change if found and not abandoned, closed
"""
rest = GerritRestAPI(url=url)
change_path = "{}~{}~{}".format(quote_plus(repo), quote_plus(branch),
change_id)
change_str = "changes/{}?o=CURRENT_REVISION".format(change_path)
change = rest.get(change_str)
try:
assert change['status'] not in 'ABANDONED' 'CLOSED', \
'Change {} is in {} state'.format(change_id, change['status'])
logging.debug('Change found: {}'.format(change))
return change
except KeyError:
logging.error('Failed to get valid change data structure from url '
'{}/{}, data returned: \n{}'.format(
change_id, change_str, change))
raise
def get_patch(change_id, repo, branch, url=con.OPENSTACK_GERRIT):
logging.info("Fetching patch for change id {}".format(change_id))
change = get_change(url, repo, branch, change_id)
if change:
current_revision = change['current_revision']
rest = GerritRestAPI(url=url)
change_path = "{}~{}~{}".format(quote_plus(repo), quote_plus(branch),
change_id)
patch_url = "changes/{}/revisions/{}/patch".format(
change_path, current_revision)
return strip_patch_sections(rest.get(patch_url))
def clone_fork(args):
ref = None
logging.info("Cloning {}".format(args.repo))
try:
cm = git.Repo(search_parent_directories=True).commit().message
except git.exc.InvalidGitRepositoryError:
logging.debug('Current Apex directory is not a git repo: {}'.format(
os.getcwd()))
cm = ''
logging.info("Current commit message: {}".format(cm))
m = re.search('{}:\s*(\S+)'.format(args.repo), cm)
if m:
change_id = m.group(1)
logging.info("Using change ID {} from {}".format(change_id, args.repo))
rest = GerritRestAPI(url=args.url)
change_str = "changes/{}?o=CURRENT_REVISION".format(change_id)
change = rest.get(change_str)
try:
assert change['status'] not in 'ABANDONED' 'CLOSED',\
'Change {} is in {} state'.format(change_id, change['status'])
if change['status'] == 'MERGED':
logging.info(
'Change {} is merged, ignoring...'.format(change_id))
else:
current_revision = change['current_revision']
ref = change['revisions'][current_revision]['ref']
logging.info('setting ref to {}'.format(ref))
except KeyError:
logging.error('Failed to get valid change data structure from url '
'{}/{}, data returned: \n{}'.format(
change_id, change_str, change))
raise
# remove existing file or directory named repo
if os.path.exists(args.repo):
if os.path.isdir(args.repo):
shutil.rmtree(args.repo)
else:
os.remove(args.repo)
ws = git.Repo.clone_from("{}/{}".format(args.url, args.repo),
args.repo,
b=args.branch)
if ref:
git_cmd = ws.git
git_cmd.fetch("{}/{}".format(args.url, args.repo), ref)
git_cmd.checkout('FETCH_HEAD')
logging.info('Checked out commit:\n{}'.format(ws.head.commit.message))
from requests.auth import HTTPDigestAuth
from pygerrit2.rest import GerritRestAPI
from pprint import pprint
import os
if __name__ == "__main__":
user = '******'
password = os.getenv('GERRIT_PASSWORD')
auth = HTTPDigestAuth(user, password)
url = 'http://gerrit.zte.com.cn'
#url = 'http://10.43.177.99'
rest = GerritRestAPI(url=url, auth=auth)
#changes=rest.get("/changes/?q=owner:self%20status:open")
#changes=rest.get("/accounts/self/groups")
#changes=rest.put("/projects/Power-RD-Projects")
#changes = rest.delete("/groups/465526c73496969c10b3412a2b76aecf5dda4df9")
changes = rest.get("/projects/")
pprint(changes)
def _main():
descr = 'Send request using Gerrit HTTP API'
parser = argparse.ArgumentParser(
description=descr,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-g',
'--gerrit-url',
dest='gerrit_url',
required=True,
help='gerrit server url')
parser.add_argument('-b',
'--basic-auth',
dest='basic_auth',
action='store_true',
help='use basic auth instead of digest')
if _kerberos_support:
parser.add_argument('-k',
'--kerberos-auth',
dest='kerberos_auth',
action='store_true',
help='use kerberos auth')
parser.add_argument('-u', '--username', dest='username', help='username')
parser.add_argument('-p', '--password', dest='password', help='password')
parser.add_argument('-n',
'--netrc',
dest='netrc',
action='store_true',
help='Use credentials from netrc')
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='store_true',
help='enable verbose (debug) logging')
options = parser.parse_args()
level = logging.DEBUG if options.verbose else logging.INFO
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=level)
if _kerberos_support and options.kerberos_auth:
if options.username or options.password \
or options.basic_auth or options.netrc:
parser.error("--kerberos-auth may not be used together with "
"--username, --password, --basic-auth or --netrc")
auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL)
elif options.username and options.password:
if options.netrc:
logging.warning("--netrc option ignored")
if options.basic_auth:
auth = HTTPBasicAuth(options.username, options.password)
else:
auth = HTTPDigestAuth(options.username, options.password)
elif options.netrc:
if options.basic_auth:
auth = HTTPBasicAuthFromNetrc(url=options.gerrit_url)
else:
auth = HTTPDigestAuthFromNetrc(url=options.gerrit_url)
else:
auth = None
rest = GerritRestAPI(url=options.gerrit_url, auth=auth)
try:
query = ["status:open"]
if auth:
query += ["owner:self"]
else:
query += ["limit:10"]
changes = rest.get("/changes/?q=%s" % "%20".join(query))
logging.info("%d changes", len(changes))
for change in changes:
logging.info(change['change_id'])
except RequestException as err:
logging.error("Error: %s", str(err))
def _main():
parser = optparse.OptionParser()
parser.add_option('-g', '--gerrit-url', dest='gerrit_url',
metavar='URL',
default=None,
help='gerrit server URL')
parser.add_option('-b', '--basic-auth', dest='basic_auth',
action='store_true',
help='(deprecated) use HTTP basic authentication instead'
' of digest')
parser.add_option('-d', '--digest-auth', dest='digest_auth',
action='store_true',
help='use HTTP digest authentication instead of basic')
parser.add_option('-n', '--dry-run', dest='dry_run',
action='store_true',
help='enable dry-run mode: show stale changes but do '
'not abandon them')
parser.add_option('-a', '--age', dest='age',
metavar='AGE',
default="6months",
help='age of change since last update '
'(default: %default)')
parser.add_option('-m', '--message', dest='message',
metavar='STRING', default=None,
help='Custom message to append to abandon message')
parser.add_option('--branch', dest='branches', metavar='BRANCH_NAME',
default=[], action='append',
help='Abandon changes only on the given branch')
parser.add_option('--exclude-branch', dest='exclude_branches',
metavar='BRANCH_NAME',
default=[],
action='append',
help='Do not abandon changes on given branch')
parser.add_option('--project', dest='projects', metavar='PROJECT_NAME',
default=[], action='append',
help='Abandon changes only on the given project')
parser.add_option('--exclude-project', dest='exclude_projects',
metavar='PROJECT_NAME',
default=[],
action='append',
help='Do not abandon changes on given project')
parser.add_option('--owner', dest='owner',
metavar='USERNAME',
default=None,
action='store',
help='Only abandon changes owned by the given user')
parser.add_option('-v', '--verbose', dest='verbose',
action='store_true',
help='enable verbose (debug) logging')
(options, _args) = parser.parse_args()
level = logging.DEBUG if options.verbose else logging.INFO
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=level)
if not options.gerrit_url:
logging.error("Gerrit URL is required")
return 1
pattern = re.compile(r"^([\d]+)(month[s]?|year[s]?|week[s]?)")
match = pattern.match(options.age)
if not match:
logging.error("Invalid age: %s", options.age)
return 1
message = "Abandoning after %s %s or more of inactivity." % \
(match.group(1), match.group(2))
if options.digest_auth:
auth_type = HTTPDigestAuthFromNetrc
else:
auth_type = HTTPBasicAuthFromNetrc
try:
auth = auth_type(url=options.gerrit_url)
gerrit = GerritRestAPI(url=options.gerrit_url, auth=auth)
except Exception as e:
logging.error(e)
return 1
logging.info(message)
try:
stale_changes = []
offset = 0
step = 500
query_terms = ["status:new", "age:%s" % options.age]
if options.branches:
query_terms += ["branch:%s" % b for b in options.branches]
elif options.exclude_branches:
query_terms += ["-branch:%s" % b for b in options.exclude_branches]
if options.projects:
query_terms += ["project:%s" % p for p in options.projects]
elif options.exclude_projects:
query_terms = ["-project:%s" % p for p in options.exclude_projects]
if options.owner:
query_terms += ["owner:%s" % options.owner]
query = "%20".join(query_terms)
while True:
q = query + "&n=%d&S=%d" % (step, offset)
logging.debug("Query: %s", q)
url = "/changes/?q=" + q
result = gerrit.get(url)
logging.debug("%d changes", len(result))
if not result:
break
stale_changes += result
last = result[-1]
if "_more_changes" in last:
logging.debug("More...")
offset += step
else:
break
except Exception as e:
logging.error(e)
return 1
abandoned = 0
errors = 0
abandon_message = message
if options.message:
abandon_message += "\n\n" + options.message
for change in stale_changes:
number = change["_number"]
try:
owner = change["owner"]["name"]
except:
owner = "Unknown"
subject = change["subject"]
if len(subject) > 70:
subject = subject[:65] + " [...]"
change_id = change["id"]
logging.info("%s (%s): %s", number, owner, subject)
if options.dry_run:
continue
try:
gerrit.post("/changes/" + change_id + "/abandon",
data='{"message" : "%s"}' % abandon_message)
abandoned += 1
except Exception as e:
errors += 1
logging.error(e)
logging.info("Total %d stale open changes", len(stale_changes))
if not options.dry_run:
logging.info("Abandoned %d changes. %d errors.", abandoned, errors)
class TestNLSR(object):
""" Test NLSR class """
# pylint: disable=too-many-instance-attributes
def __init__(self):
self.nlsr_exp_file = os.path.abspath("exp_file")
self.work_dir = os.path.abspath("work-dir")
self.exp_names = []
self.ndncxx_src = SourceManager("{}/ndn-cxx".format(self.work_dir))
self.nfd_src = SourceManager("{}/NFD".format(self.work_dir))
self.chronosync_src = SourceManager("{}/Chronosync".format(
self.work_dir))
self.psync_src = SourceManager("{}/PSync".format(self.work_dir))
self.nlsr_src = SourceManager("{}/NLSR".format(self.work_dir))
self.minindn_src = SourceManager("{}/mini-ndn".format(self.work_dir))
self.url = "https://gerrit.named-data.net"
self.auth = HTTPBasicAuthFromNetrc(self.url)
self.rest = GerritRestAPI(url=self.url, auth=self.auth)
self.rev = GerritReview()
self.message = ""
self.score = 0
self.labels = {}
self.clear_tmp()
def clear_tmp(self):
os.chdir("/tmp/minindn")
dir = [
d for d in os.listdir('/tmp/minindn')
if os.path.isdir(os.path.join('/tmp/minindn', d))
]
for f in dir:
if not f.startswith('.'):
shutil.rmtree(f)
def run_tests(self):
self.exp_names = []
with open(self.nlsr_exp_file) as test_file:
for line in test_file:
if line[0] == "#":
continue
exp = line.split(":")
test_name = exp[0]
# Run two more times if test fails
i = 0
while (i < 3):
print "Running minindn test {}".format(test_name)
print test_name
if i == 0:
self.exp_names.append(test_name)
proc = subprocess.Popen(exp[1].split())
proc.wait()
self.clear_tmp()
subprocess.call("mn --clean".split())
if proc.returncode != 0:
if i == 2:
return 1, test_name
time.sleep(30)
else:
# Test was successful
break
i += 1
time.sleep(30)
return 0, test_name
def test_nlsr(self):
""" Update and run NLSR test """
self.message = ""
if self.nlsr_src.install() != 0:
self.message = "Unable to compile NLSR"
self.score = -1
return 1
code, test = self.run_tests()
if code != 0:
print "Test {} failed!".format(test)
self.message += '\n\n'.join(self.exp_names[:len(self.exp_names) -
1])
self.message += "\n\nNLSR tester bot: Test {} failed!".format(test)
self.score = -1
return 1
else:
print "All tests passed!"
self.message = "NLSR tester bot: \n\nAll tests passed! \n\n"
self.message += '\n\n'.join(self.exp_names)
print self.message
self.score = 1
return 0
def update_dep(self):
""" Update dependencies """
git_source = [
self.ndncxx_src, self.nfd_src, self.chronosync_src, self.psync_src,
self.nlsr_src, self.minindn_src
]
for source in git_source:
if source.update_and_install() != 0:
return 1
return 0
def get_and_test_changes(self):
""" Pull the changes testable patches """
# Get open NLSR changes already verified by Jenkins and mergable and not verified by self
changes = self.rest.get(
"changes/?q=status:open+project:NLSR+branch:master+is:mergeable+label:verified+label:Verified-Integration=0"
)
#changes = self.rest.get("changes/?q=4549")
print("changes", changes)
# iterate over testable changes
for change in changes:
print "Checking patch: {}".format(change['subject'])
change_id = change['change_id']
print change_id
change_num = change['_number']
current_rev = self.rest.get(
"/changes/?q={}&o=CURRENT_REVISION".format(change_num))
#print current_rev
tmp = current_rev[0]['revisions']
for item in tmp:
patch = tmp[item]['_number']
ref = tmp[item]['ref']
print patch
print ref
# update source
if self.update_dep() != 0:
print "Unable to compile and install ndn-cxx, NFD, NLSR!"
self.rev.set_message(
"NLSR tester bot: Unable to compile dependencies!")
self.rev.add_labels({'Verified-Integration': 0})
else:
print "Pulling NLSR patch to a new branch..."
self.nlsr_src.checkout_new_branch(change_id)
self.nlsr_src.pull_from_gerrit("{}/NLSR".format(self.url), ref)
# Check if there has been a change in cpp, hpp, or wscript files
if self.nlsr_src.has_code_changes():
# Test the change
print "Testing NLSR patch"
self.test_nlsr()
print "Commenting"
self.rev.set_message(self.message)
self.rev.add_labels({'Verified-Integration': self.score})
else:
print "No change in code"
self.rev.set_message(
"NLSR tester bot: No change in code, skipped testing!")
self.rev.add_labels({'Verified-Integration': 1})
self.nlsr_src.clean_up(change_id)
print self.rev
self.rest.review(change_id, patch, self.rev)
print "\n--------------------------------------------------------\n"
time.sleep(60)
def _main():
parser = optparse.OptionParser()
parser.add_option('-g', '--gerrit-url', dest='gerrit_url',
metavar='URL',
default=None,
help='gerrit server URL')
parser.add_option('-b', '--basic-auth', dest='basic_auth',
action='store_true',
help='use HTTP basic authentication instead of digest')
parser.add_option('-n', '--dry-run', dest='dry_run',
action='store_true',
help='enable dry-run mode: show stale changes but do '
'not abandon them')
parser.add_option('-a', '--age', dest='age',
metavar='AGE',
default="6months",
help='age of change since last update '
'(default: %default)')
parser.add_option('-m', '--message', dest='message',
metavar='STRING', default=None,
help='Custom message to append to abandon message')
parser.add_option('--branch', dest='branches', metavar='BRANCH_NAME',
default=[], action='append',
help='Abandon changes only on the given branch')
parser.add_option('--exclude-branch', dest='exclude_branches',
metavar='BRANCH_NAME',
default=[],
action='append',
help='Do not abandon changes on given branch')
parser.add_option('--project', dest='projects', metavar='PROJECT_NAME',
default=[], action='append',
help='Abandon changes only on the given project')
parser.add_option('--exclude-project', dest='exclude_projects',
metavar='PROJECT_NAME',
default=[],
action='append',
help='Do not abandon changes on given project')
parser.add_option('--owner', dest='owner',
metavar='USERNAME',
default=None,
action='store',
help='Only abandon changes owned by the given user')
parser.add_option('-v', '--verbose', dest='verbose',
action='store_true',
help='enable verbose (debug) logging')
(options, _args) = parser.parse_args()
level = logging.DEBUG if options.verbose else logging.INFO
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=level)
if not options.gerrit_url:
logging.error("Gerrit URL is required")
return 1
pattern = re.compile(r"^([\d]+)(month[s]?|year[s]?|week[s]?)")
match = pattern.match(options.age)
if not match:
logging.error("Invalid age: %s", options.age)
return 1
message = "Abandoning after %s %s or more of inactivity." % \
(match.group(1), match.group(2))
if options.basic_auth:
auth_type = HTTPBasicAuthFromNetrc
else:
auth_type = HTTPDigestAuthFromNetrc
try:
auth = auth_type(url=options.gerrit_url)
gerrit = GerritRestAPI(url=options.gerrit_url, auth=auth)
except Exception as e:
logging.error(e)
return 1
logging.info(message)
try:
stale_changes = []
offset = 0
step = 500
query_terms = ["status:new", "age:%s" % options.age]
if options.branches:
query_terms += ["branch:%s" % b for b in options.branches]
elif options.exclude_branches:
query_terms += ["-branch:%s" % b for b in options.exclude_branches]
if options.projects:
query_terms += ["project:%s" % p for p in options.projects]
elif options.exclude_projects:
query_terms = ["-project:%s" % p for p in options.exclude_projects]
if options.owner:
query_terms += ["owner:%s" % options.owner]
query = "%20".join(query_terms)
while True:
q = query + "&n=%d&S=%d" % (step, offset)
logging.debug("Query: %s", q)
url = "/changes/?q=" + q
result = gerrit.get(url)
logging.debug("%d changes", len(result))
if not result:
break
stale_changes += result
last = result[-1]
if "_more_changes" in last:
logging.debug("More...")
offset += step
else:
break
except Exception as e:
logging.error(e)
return 1
abandoned = 0
errors = 0
abandon_message = message
if options.message:
abandon_message += "\n\n" + options.message
for change in stale_changes:
number = change["_number"]
try:
owner = change["owner"]["name"]
except:
owner = "Unknown"
subject = change["subject"]
if len(subject) > 70:
subject = subject[:65] + " [...]"
change_id = change["id"]
logging.info("%s (%s): %s", number, owner, subject)
if options.dry_run:
continue
try:
gerrit.post("/changes/" + change_id + "/abandon",
data='{"message" : "%s"}' % abandon_message)
abandoned += 1
except Exception as e:
errors += 1
logging.error(e)
logging.info("Total %d stale open changes", len(stale_changes))
if not options.dry_run:
logging.info("Abandoned %d changes. %d errors.", abandoned, errors)
class ReviewExtractor:
def __init__(self, url='https://gerrit.iotivity.org/gerrit/'):
self.changeList = []
self.url = url
def query(self, project_name, limit=10):
if limit is None:
limitRange = ""
else:
limitRange = "&n=" + str(limit)
self.project_name = project_name
self.limit = limit
self.rest = GerritRestAPI(url=self.url)
self.changes = self.rest.get("/changes/?q=project:" +
self.project_name + limitRange)
#self.changes = self.rest.get("/changes/?q=status:merged" + "&n=" + str(self.limit))
#print(self.changes)
#self.ownerDupList=[]
index = 1
for change in self.changes:
print("Change Collecting Index in ReviewExtractor.query() %s" %
index)
index += 1
reviewDict = self.getReviewDict(change)
if reviewDict is not None:
self.changeList.append(reviewDict)
#self.ownerDupList.append(self.changeList[-1]['changeDetail']['owner']['name'])
def getChangeOwner(self, change):
changeDetail = change['changeDetail']
changeOwner = changeDetail['owner']['name']
#return changeOwner.encode('utf-8')
return changeOwner
def getOwnerList(self, limit=None):
if limit is None:
limitRange = ""
else:
limitRange = "&n=" + str(limit)
self.ownerDupList = []
changes = self.rest.get("/changes/?q=project:" + self.project_name +
limitRange)
for index, change in enumerate(changes, 1):
print(
"Owner Collecting Index in ReviewExtractor.getOwnerList() %s" %
index)
try:
changeDetail = self.rest.get("/changes/" + change['id'] +
"/detail")
except:
print(
"Change['id'] = %s is broken while extracting detail in getOwnerList()."
% str(change['id']))
continue
else:
owner = changeDetail['owner']['name']
#self.ownerDupList.append(owner.encode('utf-8'))
self.ownerDupList.append(owner)
ownerList = set(self.ownerDupList)
return ownerList
def getProjectReviewers(self):
rl = []
for change in self.changeList:
reviewInfo = self.getReviewInfo(change)
for review in reviewInfo:
reviewer = self.getReviewer(review)
if (reviewer.startswith("jenkin")):
continue
rl.append(reviewer)
projectReviewers = set(rl)
return projectReviewers
def getReviewDict(self, change):
reviewDict = {}
try:
changeDetail = self.rest.get("/changes/" + change['id'] +
"/detail")
except:
print(
"Change['id'] = %s is broken while extracting detail in getReviewDict()."
% str(change['id']))
return None
else:
reviewDict['changeDetail'] = changeDetail
messages = changeDetail['messages']
reviewDict['messages'] = messages
comments = self.rest.get("/changes/" + change['id'] + "/comments/")
reviewDict['comments'] = comments
modifiedMessages = messages
for review in modifiedMessages:
# print("Author:"+str(change['author']['name'])+" Message:"+str(change['message']),end='\n')
review['comments'] = []
for link in comments.keys():
extraMessage = ''
isFound = False
for listElem in comments[link]:
author = listElem['author']
try:
if ((review['author']['_account_id']
== author['_account_id']) and
(review['date'] == listElem['updated'])):
isFound = True
if not link in review.keys():
review[link] = []
if not link in review['comments']:
review['comments'].append(link)
if not listElem['id'] in review[link]:
review[link].append(listElem['id'])
extraMessage = extraMessage + '\n' + 'Line ' + str(
listElem['line']
) + ': ' + listElem['message']
except:
isFound = False
if isFound:
review['message'] = review[
'message'] + '\n' + link + extraMessage + '\n'
reviewDict['reviewInfo'] = modifiedMessages
return reviewDict
def getChangeList(self):
return self.changeList
def getChangeListSubjects(self):
subjectLists = []
for change in self.changeList:
subjectLists.append(self.getChangeSubject(change))
return subjectLists
def getChangeId(self, change):
changeDetail = change['changeDetail']
return changeDetail['id']
def getChangeSubject(self, change):
changeDetail = change['changeDetail']
return changeDetail['subject']
def getChangeSubject(self, change):
changeDetail = change['changeDetail']
return changeDetail['subject']
def getLastPatchCount(self, change):
changeDetail = change['changeDetail']
message = changeDetail['messages']
return (message[-1]['_revision_number'])
def isLast(self, change, review):
changeDetail = change['changeDetail']
if (changeDetail['status'].lower() == "new"
or changeDetail['status'].lower() == "open"):
return 0
elif (self.getLastPatchCount(change) != review['_revision_number']):
return 0
else:
return 1
def getReviewInfo(self, change):
return change['reviewInfo']
def getReviewId(self, review):
return review['id']
def getReviewer(self, review):
if 'author' in review.keys():
reviewer = review['author']['name']
#return reviewer.encode('utf-8')
return reviewer
return "Gerrit-Review"
def getReview(self, review):
reviewData = review['message']
#return reviewData.encode('utf-8')
return reviewData
def getReviewDate(self, review):
return review['date']
def _main():
parser = optparse.OptionParser()
parser.add_option(
"-g",
"--gerrit-url",
dest="gerrit_url",
metavar="URL",
default=None,
help="gerrit server URL",
)
parser.add_option(
"-b",
"--basic-auth",
dest="basic_auth",
action="store_true",
help="(deprecated) use HTTP basic authentication instead of digest",
)
parser.add_option(
"-d",
"--digest-auth",
dest="digest_auth",
action="store_true",
help="use HTTP digest authentication instead of basic",
)
parser.add_option(
"-n",
"--dry-run",
dest="dry_run",
action="store_true",
help="enable dry-run mode: show stale changes but do not abandon them",
)
parser.add_option(
"-t",
"--test",
dest="testmode",
action="store_true",
help="test mode: query changes with the `test-abandon` "
"topic and ignore age option",
)
parser.add_option(
"-a",
"--age",
dest="age",
metavar="AGE",
default="6months",
help="age of change since last update in days, months"
" or years (default: %default)",
)
parser.add_option(
"-m",
"--message",
dest="message",
metavar="STRING",
default=None,
help="custom message to append to abandon message",
)
parser.add_option(
"--branch",
dest="branches",
metavar="BRANCH_NAME",
default=[],
action="append",
help="abandon changes only on the given branch",
)
parser.add_option(
"--exclude-branch",
dest="exclude_branches",
metavar="BRANCH_NAME",
default=[],
action="append",
help="do not abandon changes on given branch",
)
parser.add_option(
"--project",
dest="projects",
metavar="PROJECT_NAME",
default=[],
action="append",
help="abandon changes only on the given project",
)
parser.add_option(
"--exclude-project",
dest="exclude_projects",
metavar="PROJECT_NAME",
default=[],
action="append",
help="do not abandon changes on given project",
)
parser.add_option(
"--owner",
dest="owner",
metavar="USERNAME",
default=None,
action="store",
help="only abandon changes owned by the given user",
)
parser.add_option(
"--exclude-wip",
dest="exclude_wip",
action="store_true",
help="Exclude changes that are Work-in-Progress",
)
parser.add_option(
"-v",
"--verbose",
dest="verbose",
action="store_true",
help="enable verbose (debug) logging",
)
(options, _args) = parser.parse_args()
level = logging.DEBUG if options.verbose else logging.INFO
logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s",
level=level)
if not options.gerrit_url:
logging.error("Gerrit URL is required")
return 1
if options.testmode:
message = "Abandoning in test mode"
else:
pattern = re.compile(r"^([\d]+)(month[s]?|year[s]?|week[s]?)")
match = pattern.match(options.age)
if not match:
logging.error("Invalid age: %s", options.age)
return 1
message = "Abandoning after %s %s or more of inactivity." % (
match.group(1),
match.group(2),
)
if options.digest_auth:
auth_type = HTTPDigestAuthFromNetrc
else:
auth_type = HTTPBasicAuthFromNetrc
try:
auth = auth_type(url=options.gerrit_url)
gerrit = GerritRestAPI(url=options.gerrit_url, auth=auth)
except Exception as e:
logging.error(e)
return 1
logging.info(message)
try:
stale_changes = []
offset = 0
step = 500
if options.testmode:
query_terms = ["status:new", "owner:self", "topic:test-abandon"]
else:
query_terms = ["status:new", "age:%s" % options.age]
if options.exclude_wip:
query_terms += ["-is:wip"]
if options.branches:
query_terms += ["branch:%s" % b for b in options.branches]
elif options.exclude_branches:
query_terms += ["-branch:%s" % b for b in options.exclude_branches]
if options.projects:
query_terms += ["project:%s" % p for p in options.projects]
elif options.exclude_projects:
query_terms = ["-project:%s" % p for p in options.exclude_projects]
if options.owner and not options.testmode:
query_terms += ["owner:%s" % options.owner]
query = "%20".join(query_terms)
while True:
q = query + "&o=DETAILED_ACCOUNTS&n=%d&S=%d" % (step, offset)
logging.debug("Query: %s", q)
url = "/changes/?q=" + q
result = gerrit.get(url)
logging.debug("%d changes", len(result))
if not result:
break
stale_changes += result
last = result[-1]
if "_more_changes" in last:
logging.debug("More...")
offset += step
else:
break
except Exception as e:
logging.error(e)
return 1
abandoned = 0
errors = 0
abandon_message = message
if options.message:
abandon_message += "\n\n" + options.message
for change in stale_changes:
number = change["_number"]
project = ""
if len(options.projects) != 1:
project = "%s: " % change["project"]
owner = ""
if options.verbose:
try:
o = change["owner"]["name"]
except KeyError:
o = "Unknown"
owner = " (%s)" % o
subject = change["subject"]
if len(subject) > 70:
subject = subject[:65] + " [...]"
change_id = change["id"]
logging.info("%s%s: %s%s", number, owner, project, subject)
if options.dry_run:
continue
try:
gerrit.post(
"/changes/" + change_id + "/abandon",
json={"message": "%s" % abandon_message},
)
abandoned += 1
except Exception as e:
errors += 1
logging.error(e)
logging.info("Total %d stale open changes", len(stale_changes))
if not options.dry_run:
logging.info("Abandoned %d changes. %d errors.", abandoned, errors)
class GerritApi():
def __init__(self):
self.user = '******'
self.password = os.getenv('GERRIT_PASSWORD')
auth = HTTPDigestAuth(self.user, self.password)
#gerrit site
ip = "gerrit.zte.com.cn"
self.url = 'http://{}'.format(ip)
self.ssh_url = 'ssh://{}@{}:29418'.format(self.user, ip)
self.rest = GerritRestAPI(url=self.url, auth=auth)
self.project_info = 'projects_info.txt'
self.project_config_template = "project_config.template"
self.group_types = ('admin', 'core', 'team')
def create_projects(self):
with open(self.project_info, 'r') as f:
projects = f.readlines()
for repo in projects:
proj_repo, group = repo.split(':')
group = group.strip('\r\n')
self.create_project(proj_repo)
self.update_project_group_config(proj_repo, group)
def update_project_group_config(self, proj_repo, group):
#make groups
groups = ['{}-{}'.format(group, x) for x in self.group_types]
#set yourself the owner
#data={"owner":"10064088"}
#self.rest.put("/groups/{}/owner",data=data)
#modify project config
cmd = 'git clone {}/{}'.format(self.ssh_url, proj_repo)
os.system(cmd)
origin_folder = os.getcwd()
folder_name = proj_repo.split('/')[-1]
os.chdir(folder_name)
os.system('git fetch origin refs/meta/config:refs/meta/config')
os.system('git checkout refs/meta/config')
self.generate_groups_UUID_map(groups)
#self.generate_project_config(proj_repo.split('/')[0], group)
self.generate_project_config("Power-RD-Projects", group)
os.system('git add .')
os.system('git commit -m "update config access"')
os.system('git push origin HEAD:refs/meta/config')
#you should chdir back
os.chdir(origin_folder)
#os.system("rm -rf {}".format(folder_name))
def create_project(self, proj_repo):
#create
#you must do this replacement, or else if fails
proj_repo_id = proj_repo.replace('/', '%2F')
#you must add this! or else you cannot visit the project after created the project unless you are in administrators
payload = {"parent": "Power-RD-Projects"}
#IMPORTART: json=payload is ok, data=payload is not ok!!! f**k! wasting my debug time!
try:
ret = self.rest.put('/projects/{}'.format(proj_repo_id),
json=payload)
except:
print("project {} exists".format(proj_repo))
def generate_groups_UUID_map(self, groups):
with open('groups', 'w') as f:
for group in groups:
print(group)
ret = self.rest.get('/groups/{}'.format(group))
f.write("{}\t{}\n".format(ret['id'], group))
def generate_project_config(self, institute, group):
with open(os.path.join("..", self.project_config_template), 'r') as f:
config_template = Template(f.read())
with open('project.config', 'w') as f:
f.write(
config_template.substitute(Institute=institute, Group=group))
class DGerritHandler():
def __init__(self, gerrit_host="gerrit.mmt.com"):
self.client = None
self.ELK_HOST = "127.0.0.1:9200" # elastic search
self.index_name = datetime.datetime.now().strftime('gerrit-stats-%Y-%m')
url = "http://127.0.0.1:8080" # gerrit servers
auth = HTTPDigestAuth('admin', 'pass')
self.rest_client = GerritRestAPI(url=url, auth=auth)
# establish connection with jira
self.jira = JIRA(basic_auth=('jira', 'pass'), options = {'server': '127.0.0.1'}) # Jira server
self.regex = r'([A-Z]+-[0-9]+)'
log.info("creating a new connection with %s" % (gerrit_host))
self.client = GerritClient(gerrit_host)
log.info("Gerrit version is %s" % (self.client.gerrit_version()))
self.start_event_stream()
def start_event_stream(self):
# start listening to event stream
log.info("initiating listening to event stream")
self.client.start_event_stream()
def event_listen(self):
iter = 0
while True:
try:
elog = {}
event = self.client.get_event()
log.info("==============START=====================================")
log.info("got a new event %s -- %s" % (event.name, type(event.json)))
log.info("actual event is %s" % pformat(event.json))
elog['type'] = event.name
if event.name == "error-event":
log.info("got an error-event, exiting the script.............")
sys.exit()
elog['gerrit_id'] = event.change.number
log.info(dir(event))
if hasattr(event, 'author'):
elog['author_username'] = event.author.username if hasattr(event, 'author') else None
elog['author_email'] = event.author.email if hasattr(event, 'author') else None
elif hasattr(event, 'patchset'):
elog['author_username'] = event.patchset.author.username if hasattr(event.patchset, 'author') else None
elog['author_email'] = event.patchset.author.email if hasattr(event.patchset, 'author') else None
elog['project'] = event.change.project
elog['owner'] = event.change.owner.username
elog['branch'] = event.change.branch
elog['patchset_number'] = event.patchset.number
elog['patchset_size_deletions'] = event.json.get('patchSet').get('sizeDeletions')
elog['patchset_size_insertions'] = event.json.get('patchSet').get('sizeInsertions')
elog['subject'] = event.change.subject
if event.name == 'reviewer-added':
elog['reviewers'] = event.reviewer.username
elif event.name == 'change-merged':
elog['submitter'] = event.submitter.username
elif event.name == 'comment-added' and 'approvals' in event.json.keys():
for i in event.json.get('approvals'):
log.info(i)
if 'oldValue' in i:
log.info("----found old value----")
elog['approval_approver'] = event.author.username
elog['approver_type'] = i.get('type')
elog['approval_value'] = i.get('value')
elog['approval_description'] = i.get('description')
break
else:
elog['approval_approver'] = event.author.username
elog['approver_type'] = 'comment-added'
elog['approval_value'] = '0'
elog['approval_description'] = 'comment-added'
log.info("~~~~~~~~~~~~~~~~~~~ Start JIRA Analysis ~~~~~~~~~~~~~~~~~~~")
issue_type = []
elog['issue_id'] = re.findall(self.regex, elog['subject'])
for issue in elog['issue_id']:
issue_type.append(self.jira.issue(issue).fields.issuetype.name)
elog['issue_id_type'] = issue_type
log.info("~~~~~~~~~~~~~~~~~~~ End JIRA Analysis ~~~~~~~~~~~~~~~~~~~")
log.info("~~~~~~~~~~~~~~~~~~~ Start Gerrit File Actions ~~~~~~~~~~~~~~~~~~~")
files_info = {
"ADDED": {
"lines_added": 0,
"lines_removed": 0,
"count": 0
},
"MODIFIED": {
"lines_added": 0,
"lines_removed": 0,
"count": 0
},
"DELETED": {
"lines_added": 0,
"lines_removed": 0,
"count": 0
},
"RENAMED": {
"lines_added": 0,
"lines_removed": 0,
"count": 0
},
"COPIED": {
"lines_added": 0,
"lines_removed": 0,
"count": 0
},
"REWRITE": {
"lines_added": 0,
"lines_removed": 0,
"count": 0
}
}
query_result = self.client.run_command("query --current-patch-set --format JSON --files change:{}".format(elog['gerrit_id']))
output = query_result.stdout.read()
output = output.split('\n')
files = json.loads(output[0])
log.info(elog['project'])
for file in files['currentPatchSet']['files']:
if file['file'] not in ['/COMMIT_MSG', '/MERGE_LIST']:
files_info[file['type']]['lines_added'] += file['insertions']
files_info[file['type']]['lines_removed'] += file['deletions']
files_info[file['type']]['count'] += 1
elog['files'] = files_info
log.info("~~~~~~~~~~~~~~~~~~~ End Gerrit File Actions ~~~~~~~~~~~~~~~~~~~")
log.info("elk message %d is %s" % (iter, json.dumps(elog, indent=2)))
log.info("==============END=====================================")
self.log_to_elk(elog)
except Exception as e:
log.exception(e)
if event:
log.info(str(event.json))
finally:
iter += 1
def stop_event_stream(self):
# stop listening to gerrit event stream
log.info("stop listening to event stream")
self.client.stop_event_stream()
def log_to_elk(self, log):
log['timestamp'] = datetime.datetime.now()
elk = Elasticsearch(self.ELK_HOST)
elk.index(index=self.index_name, doc_type='gerrit_info', body=log)
def get_reviewer_list(self, change_id):
endpoint = "/changes/%s/reviewers/" % change_id
data = self.rest_client.get(endpoint)
reviewers = []
for i in data:
if i.get('username') is not None:
reviewers.append(i.get('username'))
return reviewers
" VALUES(?,?,?,?)",
[(change_id, project_name, change_subject, change_owner)])
connection.commit()
def reviewInsert(review_id, change_id, reviewer, review_date, reviewData,
usefulness):
cursor.executemany(
"INSERT INTO ReviewTable(review_id, change_id, reviewer, review_date, reviewData, usefulness)"
" VALUES(?,?,?,?,?,?)",
[(review_id, change_id, reviewer, review_date, reviewData, usefulness)
])
connection.commit()
projects = rest.get("/projects/?d")
print(projects)
print(len(projects))
changeOwnerDupList = []
for project_name in projects.keys():
print("Inserting in Project Table: " + project_name)
description = ""
if 'description' in projects[project_name].keys():
description = projects[project_name]['description']
projectInsert(project_name, projects[project_name]['id'], description)
'''
if True:
project_name='iotivity'
reviewExtractor=ReviewExtractor()
reviewExtractor.query(project_name,30)
'''
from pygerrit2.rest import GerritRestAPI
from requests.auth import HTTPDigestAuth
auth = HTTPDigestAuth('username', 'password')
query = ["status:open"]
query += ["limit:1"]
rest = GerritRestAPI(url='https://gerrit.iotivity.org/gerrit/')
#changes = rest.get("/changes/I5eefd50d6dcfcc35fd20ffd3e6e147acbf924e45/detail")
projects = rest.get("/projects/?d")
print(projects)
print(len(projects))
changeOwnerDupList = []
for change in projects:
#print(change['change_id'])
#print(change['subject'])
#print(change['status'])
changeOwnerDupList.append(change['owner']['_account_id'])
#print((changeOwnerDupList))
#changes = rest.get("/changes/" + change['change_id'] + "/detail")
#print(len(changeOwnerDupList))
changeOwnerList = []
#changeOwnerList=set(changeOwnerDupList)
#print(len(changeOwnerList))
changeOwners = []
for index, change in enumerate(projects, 1):
print("Index %s" % index)
changeDetail = rest.get("/changes/" + change['id'] + "/detail")
def _main():
descr = "Perform bulk operations on Gerrit"
parser = argparse.ArgumentParser(
description=descr, formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument(
"-u",
"--gerrit-url",
dest="url",
required=False,
default="https://gerrit-review.googlesource.com",
help="Gerrit URL",
)
parser.add_argument(
"-q", "--query", dest="query", required=True, action="store", help="query"
)
parser.add_argument(
"-o",
"--option",
dest="options",
required=False,
action="append",
help="query options",
)
parser.add_argument(
"-c",
"--commands",
dest="commands",
required=False,
action="store_true",
help="how to fetch changes; appends -o " + REVISION + " -o " + COMMANDS,
)
parser.add_argument(
"-a",
"--approve",
dest="approve",
required=False,
action="store_true",
help="apply Code-Review+2 to changes",
)
parser.add_argument(
"-v",
"--verify",
dest="verify",
required=False,
action="store_true",
help="apply Verified+1 to changes",
)
parser.add_argument(
"-s",
"--submit",
dest="submit",
required=False,
action="store_true",
help="submit changes",
)
parser.add_argument(
"-f",
"--filter",
dest="filter",
required=False,
help="filter changes by project prefix",
)
parser.add_argument(
"-fs",
"--subject",
dest="subject",
required=False,
help="filter changes by subject prefix",
)
parser.add_argument(
"--abandon",
dest="abandon",
required=False,
action="store_true",
help="abandon changes",
)
parser.add_argument(
"--hashtag",
dest="hashtags",
required=False,
action="append",
help="add hashtags",
)
parser.add_argument(
"-r",
"--reviewer",
dest="reviewers",
required=False,
action="append",
help="add reviewers",
)
options = parser.parse_args()
api = GerritRestAPI(url=options.url)
query_terms = options.query.replace(" ", "%20")
uri = "/changes/?q=" + query_terms
query_options = [o.upper() for o in options.options] if options.options else []
if options.commands:
if REVISION not in query_options:
query_options.append(REVISION)
if COMMANDS not in query_options:
query_options.append(COMMANDS)
if query_options:
uri += "".join(["&o=%s" % o for o in query_options])
changes = api.get(uri)
changes_count = len(changes)
print("Found %d changes" % changes_count)
if options.filter:
changes = [c for c in changes if c["project"].startswith(options.filter)]
if options.subject:
changes = [c for c in changes if c["subject"].startswith(options.subject)]
filtered_count = len(changes)
if filtered_count < changes_count:
print("Filtered out %d changes" % (changes_count - filtered_count))
labels = {}
review = {}
if options.reviewers:
review["reviewers"] = [{"reviewer": r} for r in options.reviewers]
if options.verify:
labels["Verified"] = 1
if options.approve:
labels["Code-Review"] = 2
if labels:
review["labels"] = labels
for change in changes:
print("%s : %s" % (change["project"], change["subject"]))
try:
if options.hashtags:
hashtags = {"add": options.hashtags}
api.post("/changes/%s/hashtags" % change["id"], json=hashtags)
if options.abandon:
api.post("/changes/%s/abandon" % change["id"])
continue
if review:
api.post(
"/changes/%s/revisions/current/review" % change["id"], json=review
)
if options.submit:
api.post("/changes/%s/submit" % change["id"])
except Exception as e:
print("Operation failed: %s" % e, file=sys.stderr)
if options.commands:
for change in changes:
repo = change["project"].split("/")[-1]
command = next(iter(change["revisions"].values()))["fetch"]["http"][
"commands"
]["Checkout"]
print("cd %s && %s && cd -" % (repo, command))
