def migration(args):
limit = int(args.limit)
offset = 0
url = URL % (limit, offset)
response = retry(requests.get, args=(url, )).json()
datasets = response['result']
while len(datasets) > 0:
for data in datasets:
Testjob = Testjobs(data['slave'], data['result'],
data['build_system_type'], data['duration'],
data['platform'], data['buildtype'],
data['testtype'], data['bugid'], data['branch'],
data['revision'], data['date'],
data['failure_classification'],
data['failures'])
try:
session.add(Testjob)
session.commit()
except Exception as error:
logging.warning(error)
session.rollback()
finally:
session.close()
# The process will move forward by set offset
offset += limit
url = URL % (limit, offset)
response = retry(requests.get, args=(url, )).json()
datasets = response['result']
def valve(head_rev, pushlog_id, branch, priority):
"""Determine which kind of job should been returned"""
priority = priority
BRANCH_COUNTER.increase_the_counter(branch)
request_list = []
try:
request_list = session.query(TaskRequests.head_rev,
TaskRequests.pushlog_id,
TaskRequests.priority).limit(40)
except Exception:
session.rollback()
requests = {}
for head_rev, pushlog_id, priority in request_list:
requests[pushlog_id] = {'head_rev': head_rev, 'priority': priority}
# If this pushlog_id has been schduled, we just return
# the priority returned before.
if pushlog_id in requests.keys():
priority = requests.get(pushlog_id)['priority']
else:
# we return all jobs for every 5 pushes.
if RequestCounter.BRANCH_COUNTER[branch] >= 5:
RequestCounter.reset(branch)
priority = None
task_request = TaskRequests(str(head_rev), str(pushlog_id), priority)
session.add(task_request)
session.commit()
return priority
def updatedb(date, platform, branch, numpushes, numjobs, sumduration):
session.query(Dailyjobs).filter(and_(Dailyjobs.date == date, Dailyjobs.branch == branch,
Dailyjobs.platform == platform)).all().delete()
session.commit()
dailyjob = Dailyjobs(date, platform, branch, numpushes, numjobs, sumduration)
try:
session.add(dailyjob)
except Exception as e:
LOG.warning(e)
session.rollback()
session.close()
def addWriters():
print request.form
writer = request.form['name']
email = request.form['email']
writer = Writers(writer, email, 0)
try:
session.add(writer)
session.commit()
except Exception as error:
print error
session.rollback()
finally:
session.close()
return redirect(url_for('writers'))
def updatedb(date, platform, branch, numpushes, numjobs, sumduration):
session.query(Dailyjobs).filter(
and_(Dailyjobs.date == date, Dailyjobs.branch == branch,
Dailyjobs.platform == platform)).all().delete()
session.commit()
dailyjob = Dailyjobs(date, platform, branch, numpushes, numjobs,
sumduration)
try:
session.add(dailyjob)
except Exception as e:
LOG.warning(e)
session.rollback()
session.close()
def dispatch():
"""
dispatch project to the writes
"""
projectName = request.form['name']
keyWords = request.form['keyWords']
description = request.form['description']
uploadPath = request.form['uploadPath']
project = Projects(name=projectName,
uploadPath=uploadPath,
keyWords=keyWords,
description=description,
updatedTime=datetime.now())
try:
session.add(project)
session.commit()
except Exception as error:
print error
session.rollback()
finally:
session.close()
return redirect(url_for('index'))
def run_seta_details_query():
buildbot = sanitize_bool(request.args.get("buildbot", 0))
branch = sanitize_string(request.args.get("branch", 'mozilla-inbound'))
taskcluster = sanitize_bool(request.args.get("taskcluster", 0))
priority = int(sanitize_string(request.args.get("priority", '5')))
jobnames = JOBSDATA.jobnames_query()
date = str(datetime.now().date())
retVal = {}
retVal[date] = []
jobtype = []
# we only support fx-team, autoland, and mozilla-inbound branch in seta
if (str(branch) in ['fx-team', 'mozilla-inbound', 'autoland']) is not True \
and str(branch) != '':
abort(404)
# For the case of TaskCluster request, we don't care which priority the user request.
# We return jobs depend on the strategy that we return high value jobs as default and
# return all jobs for every 5 push or 90 minutes for that branch.
if request.headers.get('User-Agent', '') == 'TaskCluster':
# we make taskcluster to 1 if it's a request from taskcluster, it's more reasonable and
# can simplify the request url.
taskcluster = 1
# we will return all jobs for every 90 minutes, so the return_all_jobs flag will been
# set to true if the time limit been reached.
return_all_jobs = False
# We should return full job list as a fallback, if it's a request from
# taskcluster and without head_rev or pushlog_id in there
try:
branch_info = session.query(
TaskRequests.counter, TaskRequests.datetime,
TaskRequests.reset_delta).filter(
TaskRequests.branch == branch).all()
except:
branch_info = []
time_of_now = datetime.now()
# If we got nothing related with that branch, we should create it.
if len(branch_info) == 0:
# time_of_lastreset is not a good name anyway :(
# And we treat all branches' reset_delta is 90 seconds, we should find a
# better delta for them in the further.
branch_data = TaskRequests(str(branch), 1, time_of_now,
RESET_DELTA)
try:
session.add(branch_data)
session.commit()
except Exception as error:
LOG.debug(error)
session.rollback()
finally:
session.close()
counter = 1
time_string = time_of_now
reset_delta = RESET_DELTA
# We should update it if that branch had already been stored.
else:
counter, time_string, reset_delta = branch_info[0]
counter += 1
conn = engine.connect()
statement = update(TaskRequests).where(
TaskRequests.branch == branch).values(counter=counter)
conn.execute(statement)
delta = (time_of_now - time_string).total_seconds()
# we should update the time recorder if the elapse time had
# reach the time limit of that branch.
if delta >= reset_delta:
conn = engine.connect()
statement = update(TaskRequests).where(
TaskRequests.branch == branch).values(datetime=time_of_now)
conn.execute(statement)
# we need to set the return_all_jobs flag to true.
return_all_jobs = True
# we query all jobs rather than jobs filter by the requested priority in here,
# Because we need to set the job returning strategy depend on different job priority.
query = session.query(JobPriorities.platform, JobPriorities.buildtype,
JobPriorities.testtype, JobPriorities.priority,
JobPriorities.timeout).all()
for d in query:
# we only return that job if it hasn't reach the timeout limit. And the
# timeout is zero means this job need always running.
if delta < d[4] or d[4] == 0:
# Due to the priority of all high value jobs is 1, and we
# need to return all jobs for every 5 pushes(for now).
if counter % d[3] != 0:
jobtype.append([d[0], d[1], d[2]])
# we need to return all jobs for every 90 minutes, so all jobs will been returned
# if the delta is larger than 5400
elif return_all_jobs:
jobtype.append([d[0], d[1], d[2]])
# We don't care about the timeout variable of job if it's not a taskcluster request.
else:
query = session.query(
JobPriorities.platform,
JobPriorities.buildtype,
JobPriorities.testtype,
JobPriorities.priority,
).all()
# priority = 0; run all the jobs
if priority != 1 and priority != 5:
priority = 0
# Because we store high value jobs in seta table as default,
# so we return low value jobs, means no failure related with this job as default
if priority == 0:
jobtype = JOBSDATA.jobtype_query(
) # All jobs regardless of priority
# priority =5 run all low value jobs
else:
joblist = [job for job in query if job[3] == priority]
for j in joblist:
jobtype.append([j[0], j[1], j[2]])
# TODO: filter out based on buildsystem from database, either 'buildbot' or '*'
if buildbot:
active_jobs = []
# pick up buildbot jobs from job list to faster the filter process
buildbot_jobs = [
job for job in jobnames if job['buildplatform'] == 'buildbot'
]
# find out the correspond job detail information
for job in jobtype:
for j in buildbot_jobs:
if j['name'] == job[2] and j['platform'] == job[0] and j[
'buildtype'] == job[1]:
active_jobs.append(
j['ref_data_name'] if branch is 'mozilla-inbound'
else j['ref_data_name'].
replace('mozilla-inbound', branch))
jobtype = active_jobs
# TODO: filter out based on buildsystem from database, either 'taskcluster' or '*'
if taskcluster:
active_jobs = []
taskcluster_jobs = [
job for job in jobnames if job['buildplatform'] == 'taskcluster'
]
for job in jobtype:
# we need to retranslate the jobtype back to the proper data form after all.
job[2] = job[2].replace('e10s-browser-chrome',
'browser-chrome-e10s')
job[2] = job[2].replace('e10s-devtools-chrome',
'devtools-chrome-e10s')
job[2] = job[2].replace('gl-', 'webgl-')
for j in taskcluster_jobs:
if job[2] in j['name'] and j['platform'] == job[0] and j[
'buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'])
jobtype = active_jobs
retVal[date] = jobtype
return {"jobtypes": retVal}
def uploadResults(data, branch, revision, date):
if "results" not in data:
return
results = data["results"]
count = 0
for r in results:
_id, slave, result, duration, platform, buildtype, testtype, bugid = \
'', '', '', '', '', '', '', ''
# [1468489471, u'taskcluster', u'i-0ba5dce1fab3f3768', u'?', u'unknown', u'opt', u'',
# 5945, 107, u'success', 4355877, u'-', 6689, u'gecko-decision',
# u'12626cb1-b7fc-4d8f-bcee-0ee10af509fe/0', u'Gecko Decision Task',
# u'6751f6b4d53bef7733d3063aa3f72b0832dbde74', u'gecko-decision', u'completed', 503,
# 1468489740, u'-', u'*****@*****.**',
# u'102210fe594ee9b33d82058545b1ed14f4c8206e', 1, u'D', u'scheduled', u'fill me', 1, None,
# u'-', 1468489475, u'-', u'2016-07-14T09:49:00', u'6751f6b4d53bef7733d3063aa3f72b0832dbde74', 2]
_id = r["id"]
# Skip if 'result' is unknown
result = r["result"]
if result == u'unknown':
continue
duration = '%s' % (int(r["end_timestamp"]) - int(r["start_timestamp"]))
platform = r["platform"]
if not platform:
continue
buildtype = r["platform_option"]
build_system_type = r['build_system_type']
# the testtype of builbot job is in 'ref_data_name'
# like web-platform-tests-4 in "Ubuntu VM 12.04 x64 mozilla-inbound
# but taskcluster's testtype is a part of its 'job_type_name
if r['build_system_type'] == 'buildbot':
testtype = r['ref_data_name'].split(' ')[-1]
else:
# The test name on taskcluster comes to a sort of combination
# (e.g desktop-test-linux64/debug-jittests-3)and asan job can
# been referenced as a opt job. we want the build type(debug or opt)
# to separate the job_type_name, then get "jittests-3" as testtype
# for job_type_name like desktop-test-linux64/debug-jittests-3
separator = r['platform_option'] \
if r['platform_option'] != 'asan' else 'opt'
testtype = r['job_type_name'].split(
'{buildtype}-'.format(buildtype=separator))[-1]
if r["build_system_type"] == "taskcluster":
# TODO: this is fragile, current platforms as of Jan 26, 2016 we see in taskcluster
pmap = {
"linux64": "Linux64",
"linux32": "Linux32",
"osx-10-7": "MacOSX64",
"gecko-decision": "gecko-decision",
"lint": "lint"
}
p = platform
if platform in pmap:
p = pmap[platform]
testtype = r["job_type_name"].split(p)[-1]
failure_classification = 0
try:
# http://treeherder.mozilla.org/api/failureclassification/
failure_classification = int(r["failure_classification_id"])
except ValueError:
failure_classification = 0
except TypeError:
logging.warning(
"Error, failure classification id: expecting an int, "
"but recieved %s instead" % r["failure_classification_id"])
failure_classification = 0
# Get Notes: https://treeherder.mozilla.org/api/project/mozilla-inbound/note/?job_id=5083103
if result != u'success':
url = "https://treeherder.mozilla.org/api/project/%s/note/?job_id=%s" % (
branch, _id)
try:
notes = fetch_json(url)
if notes:
bugid = notes[-1]['note']
except KeyError:
pass
# Get failure snippets: https://treeherder.mozilla.org/api/project/
# mozilla-inbound/artifact/?job_id=11651377&name=Bug+suggestions&type=json
failures = set()
if failure_classification == 2:
url = "https://treeherder.mozilla.org/api/project/%s/artifact/?job_id=%s" \
"&name=Bug+suggestions&type=json" % (branch, _id)
snippets = fetch_json(url)
if snippets:
for item in snippets[0]["blob"]:
if not item["search_terms"] and len(
item["search_terms"]) < 1:
continue
filename = item['search_terms'][0]
if (filename.endswith('.js') or filename.endswith('.xul')
or filename.endswith('.html')):
dir = item['search']
dir = (dir.split('|')[1]).strip()
if dir.endswith(filename):
dir = dir.split(filename)[0]
failures.add(dir + filename)
# https://treeherder.mozilla.org/api/project/mozilla-central/jobs/1116367/
url = "https://treeherder.mozilla.org/api/project/%s/jobs/%s/" % (
branch, _id)
data1 = fetch_json(url)
slave = data1['machine_name']
# Insert into MySQL Database
try:
testjob = Testjobs(str(slave), str(result),
str(build_system_type), str(duration),
str(platform), str(buildtype),
str(testtype), str(bugid), str(branch),
str(revision), str(date),
str(failure_classification),
str(list(failures)[0:10]))
session.add(testjob)
count += 1
session.commit()
except Exception as error:
session.rollback()
logging.warning(error)
finally:
session.close()
logging.info("uploaded %s/(%s) results for rev: %s, branch: %s, date: %s" %
(count, len(results), revision, branch, date))
def _update_job_priority_table(data):
"""Add new jobs to the priority table and update the build system if required."""
LOG.info('Fetch all rows from the job priority table.')
# Get all rows of job priorities
db_data = session.query(JobPriorities.id, JobPriorities.testtype,
JobPriorities.buildtype, JobPriorities.platform,
JobPriorities.priority, JobPriorities.timeout,
JobPriorities.expires,
JobPriorities.buildsystem).all()
# TODO: write test for this
# When the table is empty it means that we're starting the system for the first time
# and we're going to use different default values
map = {}
if not len(db_data) == 0:
priority = 1
timeout = 0
# Using %Y-%m-%d fixes this issue:
# Warning: Incorrect date value: '2016-10-28 17:36:58.153265' for column 'expires' at row 1
expiration_date = (datetime.datetime.now() +
datetime.timedelta(days=14)).strftime("%Y-%m-%d")
# Creating this data structure which will reduce how many times we iterate through the DB rows
for row in db_data:
key = tuple(row[1:4])
# This is guaranteed by a unique composite index for these 3 fields in models.py
assert key not in map,\
'"{}" should be a unique row and that is unexpected.'.format(key)
# (testtype, buildtype, platform)
map[key] = {'pk': row[0], 'build_system_type': row[7]}
else:
priority = 5
timeout = 5400
expiration_date = None
total_jobs = len(data)
new_jobs = 0
failed_changes = 0
updated_jobs = 0
# Loop through sanitized jobs, add new jobs and update the build system if needed
for job in data:
_buildsystem = job["build_system_type"]
key = _unique_key(job)
if key in map:
# We already know about this job, we might need to update the build system
row_build_system_type = map[key]['build_system_type']
if row_build_system_type == '*' or _buildsystem == '*':
# We don't need to update anything
pass
else:
# We're seeing the job again but for another build system (e.g. buildbot vs
# taskcluster). We need to change it to '*'
if row_build_system_type != _buildsystem:
_buildsystem = "*"
# Update table with new buildsystem
try:
conn = engine.connect()
statement = update(JobPriorities).where(
JobPriorities.id == map[key]['pk_key']).values(
buildsystem=_buildsystem)
conn.execute(statement)
LOG.info('Updated {}/{} from {} to {}'.format(
job['testtype'], job['platform_option'],
job['build_system_type'], _buildsystem))
updated_jobs += 1
except Exception as e:
LOG.info("key = %s, buildsystem = %s" %
(key, _buildsystem))
LOG.info("exception updating jobPriorities: %s" % e)
else:
# We have a new job from runnablejobs to add to our master list
try:
jobpriority = JobPriorities(str(job["testtype"]),
str(job["platform_option"]),
str(job["platform"]), priority,
timeout, expiration_date,
_buildsystem)
session.add(jobpriority)
session.commit()
LOG.info('New job was found ({},{},{},{})'.format(
job['testtype'],
job['platform_option'],
job['platform'],
_buildsystem,
))
new_jobs += 1
except Exception as error:
session.rollback()
LOG.warning(error)
failed_changes += 1
finally:
session.close()
LOG.info(
'We have {} new jobs and {} updated jobs out of {} total jobs processed.'
.format(new_jobs, updated_jobs, total_jobs))
if failed_changes != 0:
LOG.error(
'We have failed {} changes out of {} total jobs processed.'.format(
failed_changes, total_jobs))
def add_jobs_to_jobpriority(new_data=None,
priority=1,
timeout=0,
set_expired=False):
added_jobs = []
if not new_data:
return
# TODO: as a perf improvement we can reduce jobs prior to this expensive for loop
for job in new_data['results']:
# TODO: potentially ensure no duplicates in new_data and query once outside the loop
db_data = []
db_data = session.query(JobPriorities.id, JobPriorities.testtype,
JobPriorities.buildtype,
JobPriorities.platform, JobPriorities.priority,
JobPriorities.timeout, JobPriorities.expires,
JobPriorities.buildsystem).all()
platform = parse_platform(job['build_platform'])
if platform == None or platform == "":
continue
testtype = parse_testtype(job['build_system_type'],
job['ref_data_name'], job['platform_option'],
job['job_type_name'])
if testtype == None or testtype == "":
continue
_buildsystem = job["build_system_type"]
found = False
found_id = None
for row in db_data:
if (row[1] == testtype and row[3] == platform
and row[2] == job["platform_option"]):
#TODO: what if we have a race condition with two identical jobs
# verify the build system type is the same, or make it *
found = True
if row[7] != "*" and _buildsystem != row[7]:
_buildsystem = "*"
found_id = row[0]
# We have new jobs from runnablejobs to add to our master list
if not found:
_expired = None
if set_expired:
# set _expired = today + 14 days
# TODO: write test for it
_expired = "%s" % (datetime.datetime.now() +
datetime.timedelta(days=14))
try:
jobpriority = JobPriorities(str(testtype),
str(job["platform_option"]),
str(job["build_platform"]),
priority, timeout, _expired,
_buildsystem)
session.add(jobpriority)
session.commit()
added_jobs.append(job)
except Exception as error:
session.rollback()
logging.warning(error)
finally:
session.close()
elif _buildsystem != job['build_system_type']:
# update table with new buildsystem
conn = engine.connect()
statement = update(JobPriorities)\
.where(JobPriorities.id == found_id)\
.values(buildsystem=_buildsystem)
conn.execute(statement)
return added_jobs
def migration(args):
limit = int(args.limit)
startDate = args.startDate
offset = 0
url = URL % (startDate, limit, offset)
try:
response = retry(requests.get, args=(url, )).json()
except Exception as error:
# we will return an empty 'result' list if got exception here
logger.debug("the request to %s failed, due to %s" % (url, error))
response = {'result': []}
datasets = response['result']
session.query(Testjobs).filter(
Testjobs.date >= '%s 00:00:00' % startDate).delete()
while len(datasets) > 0:
for data in datasets:
testtype = data['testtype']
# spidermonkey builds
if len(testtype.split('pider')) > 1:
continue
# docker generation jobs
if len(testtype.split('ocker')) > 1:
continue
# hazzard builds
if len(testtype.split('az')) > 1:
continue
# decision tasks
if len(testtype.split('ecision')) > 1:
continue
# TODO: figure out a better naming strategy here
# builds, linting, etc.
if testtype.startswith('[TC]'):
continue
# skip talos jobs
if testtype in [
'chromez', 'tp5o', 'g1', 'g2', 'g3', 'g4', 'xperf',
'chromez-e10s', 'tp5o-e10s', 'g1-e10s', 'g2-e10s',
'g3-e10s', 'g4-e10s', 'xperf-e10s', 'dromaeojs',
'dromaeojs-e10s', 'svgr', 'svgr-e10s', 'remote-tsvgx',
'remote-tp4m_chrome', 'other', 'other-e10s'
]:
continue
# linter jobs
if testtype in [
'Static Analysis Opt', 'ESLint', 'Android lint', 'lint'
]:
continue
# builds
if testtype in [
'nightly', 'Android armv7 API 15+', 'ASan Dbg', 'valgrind',
'Android armv7 API 15+ Dbg', 'ASan Debug', 'pgo-build',
'ASan Opt', 'build'
]:
continue
# hidden/lower tier tests, not sure of CI system, old jobs
if testtype in [
'media-youtube-tests', 'external-media-tests',
'luciddream', 'media-tests'
]:
continue
Testjob = Testjobs(data['slave'], data['result'],
data['build_system_type'], data['duration'],
data['platform'], data['buildtype'], testtype,
data['bugid'], data['branch'], data['revision'],
data['date'], data['failure_classification'],
data['failures'])
try:
session.add(Testjob)
session.commit()
except Exception as error:
logging.warning(error)
session.rollback()
finally:
session.close()
# The process will move forward by set offset
offset += limit
url = URL % (startDate, limit, offset)
response = retry(requests.get, args=(url, )).json()
datasets = response['result']
def run_seta_details_query():
buildbot = sanitize_bool(request.args.get("buildbot", 0))
branch = sanitize_string(request.args.get("branch", 'mozilla-inbound'))
taskcluster = sanitize_bool(request.args.get("taskcluster", 0))
priority = int(sanitize_string(request.args.get("priority", '5')))
jobnames = JOBSDATA.jobnames_query()
date = str(datetime.now().date())
retVal = {}
retVal[date] = []
jobtype = []
# we only support fx-team, autoland, and mozilla-inbound branch in seta
if (str(branch) in ['fx-team', 'mozilla-inbound', 'autoland']) is not True \
and str(branch) != '':
abort(404)
# For the case of TaskCluster request, we don't care which priority the user request.
# We return jobs depend on the strategy that we return high value jobs as default and
# return all jobs for every 5 push or 90 minutes for that branch.
if request.headers.get('User-Agent', '') == 'TaskCluster':
# we make taskcluster to 1 if it's a request from taskcluster, it's more reasonable and
# can simplify the request url.
taskcluster = 1
# we will return all jobs for every 90 minutes, so the return_all_jobs flag will been
# set to true if the time limit been reached.
return_all_jobs = False
# We should return full job list as a fallback, if it's a request from
# taskcluster and without head_rev or pushlog_id in there
try:
branch_info = session.query(TaskRequests.counter,
TaskRequests.datetime,
TaskRequests.reset_delta).filter(
TaskRequests.branch == branch).all()
except:
branch_info = []
time_of_now = datetime.now()
# If we got nothing related with that branch, we should create it.
if len(branch_info) == 0:
# time_of_lastreset is not a good name anyway :(
# And we treat all branches' reset_delta is 90 seconds, we should find a
# better delta for them in the further.
branch_data = TaskRequests(str(branch), 1, time_of_now, RESET_DELTA)
try:
session.add(branch_data)
session.commit()
except Exception as error:
LOG.debug(error)
session.rollback()
finally:
session.close()
counter = 1
time_string = time_of_now
reset_delta = RESET_DELTA
# We should update it if that branch had already been stored.
else:
counter, time_string, reset_delta = branch_info[0]
counter += 1
conn = engine.connect()
statement = update(TaskRequests).where(
TaskRequests.branch == branch).values(
counter=counter)
conn.execute(statement)
delta = (time_of_now - time_string).total_seconds()
# we should update the time recorder if the elapse time had
# reach the time limit of that branch.
if delta >= reset_delta:
conn = engine.connect()
statement = update(TaskRequests).where(
TaskRequests.branch == branch).values(
datetime=time_of_now)
conn.execute(statement)
# we need to set the return_all_jobs flag to true.
return_all_jobs = True
# we query all jobs rather than jobs filter by the requested priority in here,
# Because we need to set the job returning strategy depend on different job priority.
query = session.query(JobPriorities.platform,
JobPriorities.buildtype,
JobPriorities.testtype,
JobPriorities.priority,
JobPriorities.timeout
).all()
for d in query:
# we only return that job if it hasn't reach the timeout limit. And the
# timeout is zero means this job need always running.
if delta < d[4] or d[4] == 0:
# Due to the priority of all high value jobs is 1, and we
# need to return all jobs for every 5 pushes(for now).
if counter % d[3] != 0:
jobtype.append([d[0], d[1], d[2]])
# we need to return all jobs for every 90 minutes, so all jobs will been returned
# if the delta is larger than 5400
elif return_all_jobs:
jobtype.append([d[0], d[1], d[2]])
# We don't care about the timeout variable of job if it's not a taskcluster request.
else:
query = session.query(JobPriorities.platform,
JobPriorities.buildtype,
JobPriorities.testtype,
JobPriorities.priority,
).all()
# priority = 0; run all the jobs
if priority != 1 and priority != 5:
priority = 0
# Because we store high value jobs in seta table as default,
# so we return low value jobs, means no failure related with this job as default
if priority == 0:
jobtype = JOBSDATA.jobtype_query() # All jobs regardless of priority
# priority =5 run all low value jobs
else:
joblist = [job for job in query if job[3] == priority]
for j in joblist:
jobtype.append([j[0], j[1], j[2]])
# TODO: filter out based on buildsystem from database, either 'buildbot' or '*'
if buildbot:
active_jobs = []
# pick up buildbot jobs from job list to faster the filter process
buildbot_jobs = [job for job in jobnames if job['buildplatform'] == 'buildbot']
# find out the correspond job detail information
for job in jobtype:
for j in buildbot_jobs:
if j['name'] == job[2] and j['platform'] == job[0] and j['buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'] if branch is 'mozilla-inbound'
else j['ref_data_name'].replace(
'mozilla-inbound', branch))
jobtype = active_jobs
# TODO: filter out based on buildsystem from database, either 'taskcluster' or '*'
if taskcluster:
active_jobs = []
taskcluster_jobs = [job for job in jobnames if job['buildplatform'] == 'taskcluster']
for job in jobtype:
# we need to retranslate the jobtype back to the proper data form after all.
job[2] = job[2].replace('e10s-browser-chrome', 'browser-chrome-e10s')
job[2] = job[2].replace('e10s-devtools-chrome', 'devtools-chrome-e10s')
job[2] = job[2].replace('gl-', 'webgl-')
for j in taskcluster_jobs:
if job[2] in j['name'] and j['platform'] == job[0] and j['buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'])
jobtype = active_jobs
retVal[date] = jobtype
return {"jobtypes": retVal}
def _update_job_priority_table(data):
"""Add new jobs to the priority table and update the build system if required."""
LOG.info('Fetch all rows from the job priority table.')
# Get all rows of job priorities
db_data = session.query(JobPriorities.id,
JobPriorities.testtype,
JobPriorities.buildtype,
JobPriorities.platform,
JobPriorities.priority,
JobPriorities.timeout,
JobPriorities.expires,
JobPriorities.buildsystem).all()
# TODO: write test for this
# When the table is empty it means that we're starting the system for the first time
# and we're going to use different default values
map = {}
if not len(db_data) == 0:
priority = 1
timeout = 0
# Using %Y-%m-%d fixes this issue:
# Warning: Incorrect date value: '2016-10-28 17:36:58.153265' for column 'expires' at row 1
expiration_date = (datetime.datetime.now() + datetime.timedelta(days=14)).strftime("%Y-%m-%d")
# Creating this data structure which will reduce how many times we iterate through the DB rows
for row in db_data:
key = tuple(row[1:4])
# This is guaranteed by a unique composite index for these 3 fields in models.py
assert key not in map,\
'"{}" should be a unique row and that is unexpected.'.format(key)
# (testtype, buildtype, platform)
map[key] = {'pk': row[0], 'build_system_type': row[7]}
else:
priority = 5
timeout = 5400
expiration_date = None
total_jobs = len(data)
new_jobs = 0
failed_changes = 0
updated_jobs = 0
# Loop through sanitized jobs, add new jobs and update the build system if needed
for job in data:
_buildsystem = job["build_system_type"]
key = _unique_key(job)
if key in map:
# We already know about this job, we might need to update the build system
row_build_system_type = map[key]['build_system_type']
if row_build_system_type == '*' or _buildsystem == '*':
# We don't need to update anything
pass
else:
# We're seeing the job again but for another build system (e.g. buildbot vs
# taskcluster). We need to change it to '*'
if row_build_system_type != _buildsystem:
_buildsystem = "*"
# Update table with new buildsystem
try:
conn = engine.connect()
statement = update(JobPriorities).where(
JobPriorities.id == map[key]['pk_key']).values(buildsystem=_buildsystem)
conn.execute(statement)
LOG.info('Updated {}/{} from {} to {}'.format(
job['testtype'], job['platform_option'],
job['build_system_type'], _buildsystem
))
updated_jobs += 1
except Exception as e:
LOG.info("key = %s, buildsystem = %s" % (key, _buildsystem))
LOG.info("exception updating jobPriorities: %s" % e)
else:
# We have a new job from runnablejobs to add to our master list
try:
jobpriority = JobPriorities(
str(job["testtype"]),
str(job["platform_option"]),
str(job["platform"]),
priority,
timeout,
expiration_date,
_buildsystem
)
session.add(jobpriority)
session.commit()
LOG.info('New job was found ({},{},{},{})'.format(
job['testtype'], job['platform_option'], job['platform'], _buildsystem,))
new_jobs += 1
except Exception as error:
session.rollback()
LOG.warning(error)
failed_changes += 1
finally:
session.close()
LOG.info('We have {} new jobs and {} updated jobs out of {} total jobs processed.'.format(
new_jobs, updated_jobs, total_jobs
))
if failed_changes != 0:
LOG.error('We have failed {} changes out of {} total jobs processed.'.format(
failed_changes, total_jobs
))
def uploadResults(data, branch, revision, date):
if "results" not in data:
return
results = data["results"]
count = 0
for r in results:
_id, slave, result, duration, platform, buildtype, testtype, bugid = \
'', '', '', '', '', '', '', ''
# [1468489471, u'taskcluster', u'i-0ba5dce1fab3f3768', u'?', u'unknown', u'opt', u'',
# 5945, 107, u'success', 4355877, u'-', 6689, u'gecko-decision',
# u'12626cb1-b7fc-4d8f-bcee-0ee10af509fe/0', u'Gecko Decision Task',
# u'6751f6b4d53bef7733d3063aa3f72b0832dbde74', u'gecko-decision', u'completed', 503,
# 1468489740, u'-', u'*****@*****.**',
# u'102210fe594ee9b33d82058545b1ed14f4c8206e', 1, u'D', u'scheduled', u'fill me', 1, None,
# u'-', 1468489475, u'-', u'2016-07-14T09:49:00', u'6751f6b4d53bef7733d3063aa3f72b0832dbde74', 2]
_id = r["id"]
# Skip if 'result' is unknown
result = r["result"]
if result == u'unknown':
continue
duration = '%s' % (int(r["end_timestamp"]) - int(r["start_timestamp"]))
platform = r["platform"]
if not platform:
continue
buildtype = r["platform_option"]
build_system_type = r['build_system_type']
# the testtype of builbot job is in 'ref_data_name'
# like web-platform-tests-4 in "Ubuntu VM 12.04 x64 mozilla-inbound
# but taskcluster's testtype is a part of its 'job_type_name
if r['build_system_type'] == 'buildbot':
testtype = r['ref_data_name'].split(' ')[-1]
else:
# The test name on taskcluster comes to a sort of combination
# (e.g desktop-test-linux64/debug-jittests-3)and asan job can
# been referenced as a opt job. we want the build type(debug or opt)
# to separate the job_type_name, then get "jittests-3" as testtype
# for job_type_name like desktop-test-linux64/debug-jittests-3
separator = r['platform_option'] \
if r['platform_option'] != 'asan' else 'opt'
testtype = r['job_type_name'].split(
'{buildtype}-'.format(buildtype=separator))[-1]
if r["build_system_type"] == "taskcluster":
# TODO: this is fragile, current platforms as of Jan 26, 2016 we see in taskcluster
pmap = {"linux64": "Linux64",
"linux32": "Linux32",
"osx-10-7": "MacOSX64",
"gecko-decision": "gecko-decision",
"lint": "lint"}
p = platform
if platform in pmap:
p = pmap[platform]
testtype = r["job_type_name"].split(p)[-1]
failure_classification = 0
try:
# http://treeherder.mozilla.org/api/failureclassification/
failure_classification = int(r["failure_classification_id"])
except ValueError:
failure_classification = 0
except TypeError:
logging.warning("Error, failure classification id: expecting an int, "
"but recieved %s instead" % r["failure_classification_id"])
failure_classification = 0
# Get Notes: https://treeherder.mozilla.org/api/project/mozilla-inbound/note/?job_id=5083103
if result != u'success':
url = "https://treeherder.mozilla.org/api/project/%s/note/?job_id=%s" % (branch, _id)
try:
notes = fetch_json(url)
if notes:
bugid = notes[-1]['text']
except KeyError:
if failure_classification == 2:
bugid = revision
pass
# Get failure snippets: https://treeherder.mozilla.org/api/project/
# mozilla-inbound/artifact/?job_id=11651377&name=Bug+suggestions&type=json
failures = set()
if failure_classification == 2:
url = "https://treeherder.mozilla.org/api/project/%s/artifact/?job_id=%s" \
"&name=Bug+suggestions&type=json" % (branch, _id)
snippets = fetch_json(url)
if snippets:
for item in snippets[0]["blob"]:
if not item["search_terms"] and len(item["search_terms"]) < 1:
continue
filename = item['search_terms'][0]
if (filename.endswith('.js') or filename.endswith('.xul') or
filename.endswith('.html')):
dir = item['search']
dir = (dir.split('|')[1]).strip()
if dir.endswith(filename):
dir = dir.split(filename)[0]
failures.add(dir + filename)
# https://treeherder.mozilla.org/api/project/mozilla-central/jobs/1116367/
url = "https://treeherder.mozilla.org/api/project/%s/jobs/%s/" % (branch, _id)
data1 = fetch_json(url)
slave = data1['machine_name']
# Insert into MySQL Database
try:
testjob = Testjobs(str(slave), str(result), str(build_system_type),
str(duration), str(platform), str(buildtype),
str(testtype), str(bugid), str(branch),
str(revision), str(date), str(failure_classification),
str(list(failures)[0:10]))
session.add(testjob)
count += 1
session.commit()
except Exception as error:
session.rollback()
logging.warning(error)
finally:
session.close()
logging.info("uploaded %s/(%s) results for rev: %s, branch: %s, date: %s" %
(count, len(results), revision, branch, date))