def report(self, job, build):
try:
jira = JIRA(server='https://issues.voltdb.com/', basic_auth=(JIRA_USER, JIRA_PASS), options=dict(verify=False))
except:
logging.exception('Could not connect to Jira')
return
base_url = self.jhost + '/job/' + job + '/' + str(build)
build_report = eval(self.read_url(base_url + '/api/python'))
build_result = build_report.get('result')
if build_result == "SUCCESS":
logging.info('No new issue created. Build ' + str(build) + ' resulted in: ' + build_result)
return
current_version = str(self.read_url('https://raw.githubusercontent.com/VoltDB/voltdb/master/version.txt'))
description = 'SQLCoverage failure(s) on ' + job + ', build ' + str(build) + '\n\n'
summary = job + ' : ' + build + ' : sqlcov-internal-err'
runs = build_report.get('runs')
for run in runs:
config_url = run.get('url')
config_report = eval(self.read_url(config_url + 'api/python'))
config_result = config_report.get('result')
if config_result == 'SUCCESS':
continue
config_name = config_report.get('fullDisplayName')
config_data_url = config_url + 'artifact/obj/release/sqlcoverage'
config_stats = eval(self.read_url(config_data_url + '/stats.txt'))
description += self.build_description(config_name, config_data_url, config_stats)
jenkinsbot = JenkinsBot()
jenkinsbot.create_bug_issue(JUNIT, summary, description, 'Core', current_version, ['sqlcoverage-failure', 'automatic'],
DRY_RUN=DRY_RUN)
def file_jira_issue(self, issue, DRY_RUN=False):
jenkinsbot = JenkinsBot()
error_url = issue['url']
error_report = self.read_url(error_url + '/api/python')
if error_report is None:
return
# throw a link to the query for the test history into the ticket to aid in checking/debugging the filer
note = 'http://junitstatsdb.voltdb.lan/adminer.php?server=junitstatsdb.voltdb.lan&username=qaquery&db=qa&sql=select+%2A+from+%60junit-test-failures%60+WHERE+name+%3D+%27' + issue[
'name'] + '%27+and+job+%3D+%27' + job + '%27order+by+build+desc'
# and a link to the test history on jenkins by last completed build
history = sub('/\d+/testReport/', '/lastCompletedBuild/testReport/',
error_url) + '/history/'
failed_since = error_report['failedSince']
failure_percent = issue['failurePercent']
summary = issue[
'name'] + ' is failing ~' + failure_percent + '% of the time on ' + job + ' (' + issue[
'type'] + ')'
description = error_url + '\n\n-----------------\-stack trace\----------------------------\n\n' \
+ str(error_report['errorStackTrace']) \
+ '\n\n----------------------------------------------\n\n' \
+ 'Failing since build ' + str(failed_since) + '\n' \
+ '[query history|' + note + ']\n' \
+ '!' + job + 'CountGraph.png!' \
+ '\nNOTE: this graph is from when this ticket was filed, click [here|' + history + '] for an updated graph\n'
current_version = str(
self.read_url('https://raw.githubusercontent.com/VoltDB/voltdb/'
'master/version.txt'))
new_issue_url = None
attachments = {
# filename : location
job + 'CountGraph.png':
error_url + '/history/countGraph/png?start=0&end=25'
}
try:
new_issue = jenkinsbot.create_bug_issue(
JUNIT,
summary,
description,
'Core',
current_version, ['junit-consistent-failure', 'automatic'],
attachments,
DRY_RUN=DRY_RUN)
if new_issue:
new_issue_url = "https://issues.voltdb.com/browse/" + new_issue.key
except:
logging.exception('Error with creating issue')
new_issue_url = None
return new_issue_url
def get_build_data(self, job, build_range):
"""
Gets build data for a job. Can specify an inclusive build range.
For every build specified on the job, saves the test results
:param job: Full job name on Jenkins
:param build_range: Build range that exists for the job on Jenkins, ie "700-713", "1804-1804"
"""
if job is None or build_range is None:
print('Either a job or build range was not specified.')
print(self.cmdhelp)
return
try:
builds = build_range.split('-')
build_low = int(builds[0])
build_high = int(builds[1])
if build_high < build_low:
raise Exception('Error: Left number must be lesser than or equal to right number.')
except:
logging.exception('Couldn\'t extrapolate build range.')
print(self.cmdhelp)
return
url = self.jhost + '/job/' + job + '/lastCompletedBuild/api/python'
build = self.read_url(url)
if build is None:
logging.warn('Could not retrieve last completed build. Job: %s' % job)
build = {
'number': 'unknown',
'builtOn': 'unknown'
}
latest_build = build['number']
host = build['builtOn']
issues = []
try:
db = mysql.connector.connect(host=self.dbhost, user=self.dbuser, password=self.dbpass, database='qa')
cursor = db.cursor()
except MySQLError:
logging.exception('Could not connect to qa database. User: %s. Pass: %s' % (self.dbuser, self.dbpass))
return
for build in range(build_low, build_high + 1):
build_url = self.jhost + '/job/' + job + '/' + str(build) + '/api/python'
build_report = self.read_url(build_url)
if build_report is not None:
host = build_report.get('builtOn', 'unknown')
test_url = self.jhost + '/job/' + job + '/' + str(build) + '/testReport/api/python'
test_report = self.read_url(test_url)
if test_report is None:
logging.warn(
'Could not retrieve report because url is invalid. This may be because the build %d might not '
'exist on Jenkins' % build)
logging.warn('Last completed build for this job is %s\n' % latest_build)
continue
try:
fails = test_report.get('failCount', 0)
skips = test_report.get('skipCount', 0)
passes = test_report.get('passCount', 0)
total = test_report.get('totalCount', 0)
if total == 0:
total = fails + skips + passes
if total == 0:
percent = 0
else:
percent = fails * 100.0 / total
# Get timestamp job ran on.
# Appears to be same url as build_url
job_url = self.jhost + '/job/' + job + '/' + str(build) + '/api/python'
job_report = self.read_url(job_url)
if job_report is None:
logging.warn(
'Could not retrieve report because url is invalid. This may be because the build %d might not '
'exist on Jenkins' % build)
logging.warn('Last completed build for this job is %s\n' % latest_build)
continue
# Job stamp is already in EST
job_stamp = datetime.fromtimestamp(job_report['timestamp'] / 1000).strftime('%Y-%m-%d %H:%M:%S')
# Compile job data to write to database
job_data = {
'name': job,
'timestamp': job_stamp,
'url': job_report['url'] + 'testReport',
'build': build,
'fails': fails,
'total': total,
'percent': percent
}
add_job = ('INSERT INTO `junit-builds` '
'(name, stamp, url, build, fails, total, percent) '
'VALUES (%(name)s, %(timestamp)s, %(url)s, %(build)s, %(fails)s, %(total)s, %(percent)s)')
try:
cursor.execute(add_job, job_data)
db.commit()
except MySQLError:
logging.exception('Could not add job data to database')
# Some of the test results are structured differently, depending on the matrix configurations.
child_reports = test_report.get('childReports', None)
if child_reports is None:
child_reports = [
{
'result': test_report,
'child': {
'url': test_url.replace('testReport/api/python', '')
}
}
]
# Traverse through reports into test suites and get failed test case data to write to database.
for child in child_reports:
suites = child['result']['suites']
for suite in suites:
cases = suite['cases']
test_stamp = suite.get('timestamp', None)
if test_stamp is None or test_stamp == 'None':
test_stamp = job_stamp
else:
# Create datetime object from test_stamp string
test_stamp = datetime.strptime(test_stamp, '%Y-%m-%dT%H:%M:%S')
# Convert test stamp from GMT to EST and store as string
test_stamp = (test_stamp - timedelta(hours=4)).strftime('%Y-%m-%d %H:%M:%S')
for case in cases:
name = case['className'] + '.' + case['name']
status = case['status']
testcase_url = child['child']['url'] + 'testReport/' + name
testcase_url.replace('.test', '/test').replace('.Test', '/Test').replace('-', '_')
# Record tests that don't pass.
if status != 'PASSED':
test_data = {
'name': name,
'job': job,
'status': status,
'timestamp': test_stamp,
'url': testcase_url,
'build': build,
'host': host
}
if status == 'FAILED':
issues.append(test_data)
add_test = ('INSERT INTO `junit-test-failures` '
'(name, job, status, stamp, url, build, host) '
'VALUES (%(name)s, %(job)s, %(status)s, %(timestamp)s, '
'%(url)s, %(build)s, %(host)s)')
try:
cursor.execute(add_test, test_data)
db.commit()
except MySQLError:
logging.exception('Could not add test data to database')
except KeyError:
logging.exception('Error retrieving test data for this particular build: %d\n' % build)
except Exception:
# Catch all errors to avoid causing a failing build for the upstream job in case this is being
# called from the junit-test-branch on Jenkins
logging.exception('Catching unexpected errors to avoid causing a failing build for the upstream job')
cursor.close()
db.close()
if job != PRO and job != COMMUNITY:
return
try:
jenkinsbot = JenkinsBot()
for issue in issues:
# Only report pro and community job
error_url = issue['url']
error_report = self.read_url(error_url + '/api/python')
if error_report is None:
continue
age = error_report['age']
yesterday = datetime.now() - timedelta(days=1)
timestamp = issue['timestamp']
old = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S') < yesterday
# Don't file ticket if age within tolerance and this failure happened over one day ago
if age < TOLERANCE and old:
continue
failed_since = error_report['failedSince']
summary = issue['name'] + ' is failing since build ' + str(failed_since) + ' on ' + job
description = error_url + '\n' + error_report['errorStackTrace']
current_version = str(self.read_url('https://raw.githubusercontent.com/VoltDB/voltdb/'
'master/version.txt'))
jenkinsbot.create_bug_issue(JUNIT, summary, description, 'Core', current_version,
['junit-consistent-failure', 'automatic'])
except:
logging.exception('Error with creating issue')
def get_build_data(self, job, build_range):
"""
Gets build data for a job. Can specify an inclusive build range.
For every build specified on the job, saves the test results
:param job: Full job name on Jenkins
:param build_range: Build range that exists for the job on Jenkins, ie "700-713", "1804-1804"
"""
if job is None or build_range is None:
print('Either a job or build range was not specified.')
print(self.cmdhelp)
return
try:
builds = build_range.split('-')
build_low = int(builds[0])
build_high = int(builds[1])
if build_high < build_low:
raise Exception('Error: Left number must be lesser than or equal to right number.')
except:
logging.exception('Couldn\'t extrapolate build range.')
print(self.cmdhelp)
return
url = self.jhost + '/job/' + job + '/lastCompletedBuild/api/python'
build = self.read_url(url)
if build is None:
logging.warn('Could not retrieve last completed build. Job: %s' % job)
build = {
'number': 'unknown',
'builtOn': 'unknown'
}
latest_build = build['number']
host = build['builtOn']
issues = []
try:
db = mysql.connector.connect(host=self.dbhost, user=self.dbuser, password=self.dbpass, database='qa')
cursor = db.cursor()
except MySQLError:
logging.exception('Could not connect to qa database. User: %s. Pass: %s' % (self.dbuser, self.dbpass))
return
for build in range(build_low, build_high + 1):
build_url = self.jhost + '/job/' + job + '/' + str(build) + '/api/python'
build_report = self.read_url(build_url)
if build_report is not None:
host = build_report.get('builtOn', 'unknown')
test_url = self.jhost + '/job/' + job + '/' + str(build) + '/testReport/api/python'
test_report = self.read_url(test_url)
if test_report is None:
logging.warn(
'Could not retrieve report because url is invalid. This may be because the build %d might not '
'exist on Jenkins' % build)
logging.warn('Last completed build for this job is %s\n' % latest_build)
continue
try:
fails = test_report.get('failCount', 0)
skips = test_report.get('skipCount', 0)
passes = test_report.get('passCount', 0)
total = test_report.get('totalCount', 0)
if total == 0:
total = fails + skips + passes
if total == 0:
percent = 0
else:
percent = fails * 100.0 / total
# Get timestamp job ran on.
# Appears to be same url as build_url
job_url = self.jhost + '/job/' + job + '/' + str(build) + '/api/python'
job_report = self.read_url(job_url)
if job_report is None:
logging.warn(
'Could not retrieve report because url is invalid. This may be because the build %d might not '
'exist on Jenkins' % build)
logging.warn('Last completed build for this job is %s\n' % latest_build)
continue
# Job stamp is already in EST
job_stamp = datetime.fromtimestamp(job_report['timestamp'] / 1000).strftime('%Y-%m-%d %H:%M:%S')
# Compile job data to write to database
job_data = {
'name': job,
'timestamp': job_stamp,
'url': job_report['url'] + 'testReport',
'build': build,
'fails': fails,
'total': total,
'percent': percent
}
add_job = ('INSERT INTO `junit-builds` '
'(name, stamp, url, build, fails, total, percent) '
'VALUES (%(name)s, %(timestamp)s, %(url)s, %(build)s, %(fails)s, %(total)s, %(percent)s)')
try:
cursor.execute(add_job, job_data)
db.commit()
except MySQLError:
logging.exception('Could not add job data to database')
# Some of the test results are structured differently, depending on the matrix configurations.
child_reports = test_report.get('childReports', None)
if child_reports is None:
child_reports = [
{
'result': test_report,
'child': {
'url': test_url.replace('testReport/api/python', '')
}
}
]
# Traverse through reports into test suites and get failed test case data to write to database.
for child in child_reports:
suites = child['result']['suites']
for suite in suites:
cases = suite['cases']
test_stamp = suite.get('timestamp', None)
if test_stamp is None or test_stamp == 'None':
test_stamp = job_stamp
else:
# Create datetime object from test_stamp string
test_stamp = datetime.strptime(test_stamp, '%Y-%m-%dT%H:%M:%S')
# Convert test stamp from GMT to EST and store as string
test_stamp = (test_stamp - timedelta(hours=4)).strftime('%Y-%m-%d %H:%M:%S')
for case in cases:
name = case['className'] + '.' + case['name']
status = case['status']
url_name = name.replace('.test', '/test').replace('.Test', '/Test').replace('-', '_')
if "vdm-py-test" in child['child']['url']:
testcase_url = child['child']['url'] + 'testReport/' + '(root)/' + url_name
else:
testcase_url = child['child']['url'] + 'testReport/' + url_name
# Record tests that don't pass.
if status != 'PASSED':
test_data = {
'name': name,
'job': job,
'status': status,
'timestamp': test_stamp,
'url': testcase_url,
'build': build,
'host': host
}
if status == 'FAILED' and (job == PRO or job == COMMUNITY):
#1. query to see if we already have one failure in same job
#if we don't, skip filing the ticket
params1 = {
'name': test_data['name'],
'job_number': test_data['build'],
'job': test_data['job']
}
query1 = ("""
SELECT count(*)
FROM `junit-test-failures`
WHERE job=%(job)s
AND name = %(name)s
AND build = %(job_number)s-1
AND status = 'FAILED';
""")
cursor.execute(query1,params1)
output1 = cursor.fetchone()
output1 = output1[0]
# 2. query to see if we already have two failures for same test in alt-job type
#and if we do skip filing the ticket.
params2 = {
'name': test_data['name']
}
if test_data['job'] == PRO:
params2['job'] = COMMUNITY
elif test_data['job'] == COMMUNITY:
params2['job'] = PRO
query2 = ("""
SELECT COUNT(*) AS fails
FROM
(SELECT DISTINCT job,
name,
status,
build,
stamp
FROM `junit-test-failures`
WHERE job=%(job)s
AND name = %(name)s
ORDER BY stamp DESC
LIMIT 2) AS t
WHERE t.status = 'FAILED';
""")
cursor.execute(query2,params2)
output2 = cursor.fetchone()
output2 = output2[0]
# if next most recent in job is a fail and two most recent in alt-job are not fails
if output1 ==1 and output2 != 2:
issues.append(test_data)
add_test = ('INSERT INTO `junit-test-failures` '
'(name, job, status, stamp, url, build, host) '
'VALUES (%(name)s, %(job)s, %(status)s, %(timestamp)s, '
'%(url)s, %(build)s, %(host)s)')
try:
cursor.execute(add_test, test_data)
db.commit()
except MySQLError:
logging.exception('Could not add test data to database')
except KeyError:
logging.exception('Error retrieving test data for this particular build: %d\n' % build)
except Exception:
# Catch all errors to avoid causing a failing build for the upstream job in case this is being
# called from the junit-test-branch on Jenkins
logging.exception('Catching unexpected errors to avoid causing a failing build for the upstream job')
cursor.close()
db.close()
if job != PRO and job != COMMUNITY:
return
try:
jenkinsbot = JenkinsBot()
for issue in issues:
# Only report pro and community job
error_url = issue['url']
error_report = self.read_url(error_url + '/api/python')
if error_report is None:
continue
age = error_report['age']
yesterday = datetime.now() - timedelta(days=1)
timestamp = issue['timestamp']
old = datetime.strptime(timestamp, '%Y-%m-%d %H:%M:%S') < yesterday
failed_since = error_report['failedSince']
summary = issue['name'] + ' is failing since build ' + str(failed_since) + ' on ' + job
description = error_url + '\n' + str(error_report['errorStackTrace'])
current_version = str(self.read_url('https://raw.githubusercontent.com/VoltDB/voltdb/'
'master/version.txt'))
jenkinsbot.create_bug_issue(JUNIT, summary, description, 'Core', current_version, ['junit-consistent-failure', 'automatic'])
except:
logging.exception('Error with creating issue')
def build_history(self, job, build_range):
"""
Displays build history for a job. Can specify an inclusive build range.
For every build specified on the job, saves the test results
:param job: Full job name on Jenkins
:param build_range: Build range that exists for the job on Jenkins, ie "700-713", "1804-1804"
"""
if job is None or build_range is None:
print('Either a job or build range was not specified.')
print(self.cmdhelp)
return
try:
builds = build_range.split('-')
build_low = int(builds[0])
build_high = int(builds[1])
if build_high < build_low:
raise Exception('Error: Left number must be lesser than or equal to right number.')
except:
logging.exception('Couldn\'t extrapolate build range.')
print(self.cmdhelp)
return
url = self.jhost + '/job/' + job + '/lastCompletedBuild/api/python'
build = self.read_url(url)
if build is None:
logging.warn('Could not retrieve last completed build. Job: %s' % job)
build = {
'number': 'unknown',
'builtOn': 'unknown'
}
latest_build = build['number']
host = build['builtOn']
issues = []
try:
db = mysql.connector.connect(host=self.dbhost, user=self.dbuser, password=self.dbpass, database='qa')
cursor = db.cursor()
except MySQLError:
logging.exception('Could not connect to qa database. User: %s. Pass: %s' % (self.dbuser, self.dbpass))
return
for build in range(build_low, build_high + 1):
build_url = self.jhost + '/job/' + job + '/' + str(build) + '/api/python'
build_report = self.read_url(build_url)
if build_report is not None:
host = build_report.get('builtOn', 'unknown')
test_url = self.jhost + '/job/' + job + '/' + str(build) + '/testReport/api/python'
test_report = self.read_url(test_url)
if test_report is None:
logging.warn(
'Could not retrieve report because url is invalid. This may be because the build %d might not '
'exist on Jenkins' % build)
logging.warn('Last completed build for this job is %s\n' % latest_build)
continue
try:
fails = test_report.get('failCount', 0)
skips = test_report.get('skipCount', 0)
passes = test_report.get('passCount', 0)
total = test_report.get('totalCount', 0)
if total == 0:
total = fails + skips + passes
if total == 0:
percent = 0
else:
percent = fails * 100.0 / total
# Get timestamp job ran on.
job_url = self.jhost + '/job/' + job + '/' + str(build) + '/api/python'
job_report = self.read_url(job_url)
if job_report is None:
logging.warn(
'Could not retrieve report because url is invalid. This may be because the build %d might not '
'exist on Jenkins' % build)
logging.warn('Last completed build for this job is %s\n' % latest_build)
continue
# Job stamp is already in EST
job_stamp = datetime.fromtimestamp(job_report['timestamp'] / 1000).strftime('%Y-%m-%d %H:%M:%S')
# Compile job data to write to database
job_data = {
'name': job,
'timestamp': job_stamp,
'url': job_report['url'] + 'testReport',
'build': build,
'fails': fails,
'total': total,
'percent': percent
}
add_job = ('INSERT INTO `junit-builds` '
'(name, stamp, url, build, fails, total, percent) '
'VALUES (%(name)s, %(timestamp)s, %(url)s, %(build)s, %(fails)s, %(total)s, %(percent)s)')
try:
cursor.execute(add_job, job_data)
db.commit()
except MySQLError:
logging.exception('Could not add job data to database')
# Some of the test results are structured differently, depending on the matrix configurations.
child_reports = test_report.get('childReports', None)
if child_reports is None:
child_reports = [
{
'result': test_report,
'child': {
'url': test_url.replace('testReport/api/python', '')
}
}
]
# Traverse through reports into test suites and get failed test case data to write to database.
for child in child_reports:
suites = child['result']['suites']
for suite in suites:
cases = suite['cases']
test_stamp = suite.get('timestamp', None)
if test_stamp is None or test_stamp == 'None':
test_stamp = job_stamp
else:
# Create datetime object from test_stamp string
test_stamp = datetime.strptime(test_stamp, '%Y-%m-%dT%H:%M:%S')
# Convert test stamp from GMT to EST and store as string
test_stamp = (test_stamp - timedelta(hours=4)).strftime('%Y-%m-%d %H:%M:%S')
for case in cases:
name = case['className'] + '.' + case['name']
status = case['status']
report_url = child['child']['url'] + 'testReport/'
report_url = report_url + \
name.replace('.test', '/test').replace('.Test', '/Test').replace('-', '_')
# Record tests that don't pass.
if status != 'PASSED':
test_data = {
'name': name,
'job': job,
'status': status,
'timestamp': test_stamp,
'url': report_url,
'build': build,
'host': host
}
if status == 'FAILED':
issues.append(test_data)
add_test = ('INSERT INTO `junit-test-failures` '
'(name, job, status, stamp, url, build, host) '
'VALUES (%(name)s, %(job)s, %(status)s, %(timestamp)s, '
'%(url)s, %(build)s, %(host)s)')
try:
cursor.execute(add_test, test_data)
db.commit()
except MySQLError:
logging.exception('Could not add test data to database')
except KeyError:
logging.exception('Error retrieving test data for this particular build: %d\n' % build)
except Exception:
# Catch all errors to avoid causing a failing build for the upstream job in case this is being
# called from the junit-test-branch on Jenkins
logging.exception('Catching unexpected errors to avoid causing a failing build for the upstream job')
cursor.close()
db.close()
try:
jenkinsbot = JenkinsBot()
for issue in issues:
error_url = issue['url']
error_report = self.read_url(error_url)
if error_report is not None:
age = error_report['age']
summary = issue['name'] + ' has failed ' + str(age) + ' times in a row on ' + issue['job']
description = error_url + '\n' + error_report['errorStackTrace']
jenkinsbot.create_bug_issue(age, issue, summary, description, 'Core', 'V6.6',
'junit-consistent-failure')
except:
logging.exception('Error with creating issue')