def delete_product():
print("Available products are following: ")
Product.browse_product()
p_id = input("\nEnter the product id which you what to delete: ")
db_session.query(Product).filter(Product.id == p_id).delete()
db_session.commit()
db_session.close()
def clearResults(branch, startdate):
date_xx_days_ago = datetime.date.today() - datetime.timedelta(days=180)
session.query(Testjobs).filter(branch == branch).\
filter(and_(Testjobs.date >= startdate), (Testjobs.date < date_xx_days_ago)).\
delete(synchronize_session='fetch')
session.commit()
def clearResults(branch, startdate):
date_xx_days_ago = datetime.date.today() - datetime.timedelta(days=180)
session.query(Testjobs).filter(branch == branch).\
filter(and_(Testjobs.date >= startdate), (Testjobs.date < date_xx_days_ago)).\
delete(synchronize_session='fetch')
session.commit()
def run_seta_summary_query():
query = session.query(Seta.date).distinct().all()
retVal = {}
dates = []
for d in query:
dates.append(d[0].strftime("%Y-%m-%d"))
for d in dates:
count = session.query(Seta.id).filter(Seta.date == d).count()
retVal['%s' % d] = "%s" % int(count)
return {'dates': retVal}
def updatedb(date, platform, branch, numpushes, numjobs, sumduration):
session.query(Dailyjobs).filter(and_(Dailyjobs.date == date, Dailyjobs.branch == branch,
Dailyjobs.platform == platform)).all().delete()
session.commit()
dailyjob = Dailyjobs(date, platform, branch, numpushes, numjobs, sumduration)
try:
session.add(dailyjob)
except Exception as e:
LOG.warning(e)
session.rollback()
session.close()
def insert_in_database(to_insert, date=None):
if not date:
date = datetime.datetime.now().strftime('%Y-%m-%d')
else:
date = date.strftime('%Y-%m-%d')
session.query(Seta).filter(Seta.date == date).delete(
synchronize_session='fetch')
session.commit()
for jobtype in to_insert:
job = Seta(str(jobtype), date)
session.add(job)
session.commit()
session.close()
def updatedb(date, platform, branch, numpushes, numjobs, sumduration):
session.query(Dailyjobs).filter(
and_(Dailyjobs.date == date, Dailyjobs.branch == branch,
Dailyjobs.platform == platform)).all().delete()
session.commit()
dailyjob = Dailyjobs(date, platform, branch, numpushes, numjobs,
sumduration)
try:
session.add(dailyjob)
except Exception as e:
LOG.warning(e)
session.rollback()
session.close()
def update_jobpriorities(to_insert, _priority, _timeout):
# to_insert is currently high priority, pri=1 jobs, all else are pri=5 jobs
changed_jobs = []
for item in to_insert:
# NOTE: we ignore JobPriorities with expires as they take precendence
data = session.query(JobPriorities.id, JobPriorities.priority)\
.filter(and_(JobPriorities.testtype == item[2],
JobPriorities.buildtype == item[1],
JobPriorities.platform == item[0],
JobPriorities.expires == None)).all()
if len(data) != 1:
# TODO: if 0 items, do we add the job? if >1 do we alert and cleanup?
continue
if data[0][1] != _priority:
changed_jobs.append(item)
conn = engine.connect()
statement = update(JobPriorities)\
.where(and_(JobPriorities.testtype == item[2],
JobPriorities.buildtype == item[1],
JobPriorities.platform == item[0]))\
.values(priority=_priority,
timeout=_timeout)
conn.execute(statement)
return changed_jobs
def retrievedb(branch, date):
data = session.query(Testjobs.revision, func.count(Testjobs.result),
func.sum(Testjobs.duration), Testjobs.platform).\
filter(and_(Testjobs.branch == branch, Testjobs.date.like(date),
~Testjobs.testtype.like('build'), Testjobs.testtype != 'valgrind')).group_by(
Testjobs.revision, Testjobs.platform).all()
for rows in data:
revision = rows[0]
jobs = int(rows[1])
duration = int(rows[2])
platform = rows[3]
found = False
for p in platforms:
if platform.startswith(p):
platform = p
found = True
break
# we can skip b2g/mulet or other platforms that don't match our core set
if not found:
continue
if revision not in revisions_dict:
revisions_dict[revision] = {}
for p in platforms:
revisions_dict[revision][p] = [date, 0, 0]
revisions_dict[revision][platform][1] += jobs
revisions_dict[revision][platform][2] += duration
def increase_jobs_priority(high_value_jobs, priority=1, timeout=0):
"""For every high value job try to see if we need to update increase its priority
Currently, high value jobs have a priority of 1 and a timeout of 0.
Return how many jobs had their priority increased
"""
changed_jobs = []
for item in high_value_jobs:
# NOTE: we ignore JobPriorities with expires as they take precendence
data = session.query(JobPriorities.id, JobPriorities.priority)\
.filter(and_(JobPriorities.testtype == item[2],
JobPriorities.buildtype == item[1],
JobPriorities.platform == item[0],
JobPriorities.expires == None)).all() # flake8: noqa
if len(data) != 1:
# TODO: if 0 items, do we add the job? if >1 do we alert and cleanup?
continue
if data[0][1] != priority:
changed_jobs.append(item)
conn = engine.connect()
statement = update(JobPriorities)\
.where(and_(JobPriorities.testtype == item[2],
JobPriorities.buildtype == item[1],
JobPriorities.platform == item[0]))\
.values(priority=priority, timeout=timeout)
conn.execute(statement)
return changed_jobs
def detail():
"""
Detail page of project
"""
id = request.args.get("id")
projectDetail = session.query(Projects).filter(Projects.id == id).all()
return render_template("detail.html", projectDetail=projectDetail)
def retrievedb(branch, date):
data = session.query(Testjobs.revision, func.count(Testjobs.result),
func.sum(Testjobs.duration), Testjobs.platform).\
filter(and_(Testjobs.branch == branch, Testjobs.date.like(date),
~Testjobs.testtype.like('build'), Testjobs.testtype != 'valgrind')).group_by(
Testjobs.revision, Testjobs.platform).all()
for rows in data:
revision = rows[0]
jobs = int(rows[1])
duration = int(rows[2])
platform = rows[3]
found = False
for p in platforms:
if platform.startswith(p):
platform = p
found = True
break
# we can skip b2g/mulet or other platforms that don't match our core set
if not found:
continue
if revision not in revisions_dict:
revisions_dict[revision] = {}
for p in platforms:
revisions_dict[revision][p] = [date, 0, 0]
revisions_dict[revision][platform][1] += jobs
revisions_dict[revision][platform][2] += duration
def valve(head_rev, pushlog_id, branch, priority):
"""Determine which kind of job should been returned"""
priority = priority
BRANCH_COUNTER.increase_the_counter(branch)
request_list = []
try:
request_list = session.query(TaskRequests.head_rev,
TaskRequests.pushlog_id,
TaskRequests.priority).limit(40)
except Exception:
session.rollback()
requests = {}
for head_rev, pushlog_id, priority in request_list:
requests[pushlog_id] = {'head_rev': head_rev, 'priority': priority}
# If this pushlog_id has been schduled, we just return
# the priority returned before.
if pushlog_id in requests.keys():
priority = requests.get(pushlog_id)['priority']
else:
# we return all jobs for every 5 pushes.
if RequestCounter.BRANCH_COUNTER[branch] >= 5:
RequestCounter.reset(branch)
priority = None
task_request = TaskRequests(str(head_rev), str(pushlog_id), priority)
session.add(task_request)
session.commit()
return priority
def run_dailyjob_query():
start_date, end_date = clean_date_params(request.args)
start_date = start_date - timedelta(days=1)
end_date = end_date + timedelta(days=1)
data = session.query(Dailyjobs.date, Dailyjobs.platform, Dailyjobs.branch, Dailyjobs.numjobs,
Dailyjobs.sumduration).\
filter(Dailyjobs.date.between(start_date, end_date)).order_by(case([
(Dailyjobs.platform == 'linux', 1),
(Dailyjobs.platform == 'osx', 2),
(Dailyjobs.platform == 'win', 3),
(Dailyjobs.platform == 'android', 4)], else_='5')).all()
output = {}
for rows in data:
date = str(rows[0])
platform = rows[1]
branch = rows[2]
numpushes = int(rows[3])
numjobs = int(rows[4])
sumduration = int(rows[5])
if date not in output:
output[date] = {'mozilla-inbound': [], 'fx-team': [], 'try': [], 'autoland': []}
if 'mozilla-inbound' in branch:
output[date]['mozilla-inbound'].append([platform, numpushes, numjobs, sumduration])
elif 'fx-team' in branch:
output[date]['fx-team'].append([platform, numpushes, numjobs, sumduration])
elif 'try' in branch:
output[date]['try'].append([platform, numpushes, numjobs, sumduration])
elif 'autoland' in branch:
output[date]['autoland'].append([platform, numpushes, numjobs, sumduration])
return {'dailyjobs': output}
def run_dailyjob_query():
start_date, end_date = clean_date_params(request.args)
start_date = start_date - timedelta(days=1)
end_date = end_date + timedelta(days=1)
data = session.query(Dailyjobs.date, Dailyjobs.platform, Dailyjobs.branch, Dailyjobs.numjobs,
Dailyjobs.sumduration).\
filter(Dailyjobs.date.between(start_date, end_date)).order_by(case([
(Dailyjobs.platform == 'linux', 1),
(Dailyjobs.platform == 'osx', 2),
(Dailyjobs.platform == 'win', 3),
(Dailyjobs.platform == 'android', 4)], else_='5')).all()
output = {}
for rows in data:
date = str(rows[0])
platform = rows[1]
branch = rows[2]
numpushes = int(rows[3])
numjobs = int(rows[4])
sumduration = int(rows[5])
if date not in output:
output[date] = {'mozilla-inbound': [], 'fx-team': [], 'try': [], 'autoland': []}
if 'mozilla-inbound' in branch:
output[date]['mozilla-inbound'].append([platform, numpushes, numjobs, sumduration])
elif 'fx-team' in branch:
output[date]['fx-team'].append([platform, numpushes, numjobs, sumduration])
elif 'try' in branch:
output[date]['try'].append([platform, numpushes, numjobs, sumduration])
elif 'autoland' in branch:
output[date]['autoland'].append([platform, numpushes, numjobs, sumduration])
return {'dailyjobs': output}
def check_data(query_date):
ret_val = []
data = session.query(Seta.jobtype).filter(Seta.date == query_date).all()
if not data:
print "The database does not have data for the given %s date." % query_date
for date in range(-3, 4):
current_date = query_date + datetime.timedelta(date)
jobtype = session.query(Seta).filter(Seta.date == current_date)
if jobtype:
print "The data is available for date=%s" % current_date
return ret_val
for job in data:
parts = job[0].split("'")
ret_val.append("%s" % [str(parts[1]), str(parts[3]), str(parts[5])])
return ret_val
def run_seta_query():
start_date, end_date = clean_date_params(request.args, delta=SETA_WINDOW)
# we would like to enlarge the datetime range to make sure the latest failures been get.
start_date = start_date - timedelta(days=1)
end_date = end_date + timedelta(days=1)
data = session.query(Testjobs.bugid, Testjobs.platform, Testjobs.buildtype, Testjobs.testtype,
Testjobs.duration).filter(and_(Testjobs.failure_classification == 2,
Testjobs.date >= start_date,
Testjobs.date <= end_date)).all()
failures = {}
for d in data:
failures.setdefault(d[0], []).append(d[1:])
return {'failures': failures}
def run_seta_query():
start_date, end_date = clean_date_params(request.args, delta=SETA_WINDOW)
# we would like to enlarge the datetime range to make sure the latest failures been get.
start_date = start_date - timedelta(days=1)
end_date = end_date + timedelta(days=1)
data = session.query(Testjobs.bugid, Testjobs.platform, Testjobs.buildtype, Testjobs.testtype,
Testjobs.duration).filter(and_(Testjobs.failure_classification == 2,
Testjobs.date >= start_date,
Testjobs.date <= end_date)).all()
failures = {}
for d in data:
failures.setdefault(d[0], []).append(d[1:])
return {'failures': failures}
def run_results_day_flot_query():
"""
This function returns the total failures/total jobs data per day for all platforms.
It is sending the data in the format required by flot.Flot is a jQuery package used
for 'attractive' plotting
"""
start_date, end_date = clean_date_params(request.args)
platforms = ['android4.0',
'android2.3',
'linux32',
'winxp',
'win7',
'win8',
'osx10.6',
'osx10.7',
'osx10.8']
data_platforms = {}
for platform in platforms:
query_results = session.query(Testjobs.date.label('day'),
func.count(Testjobs.result == 'testfailed'
).label("failures"),
func.count(Testjobs).label('totals')).filter(
and_(Testjobs.platform == platform,
Testjobs.date >= start_date, Testjobs.date <= end_date)).group_by('day').all()
dates = []
data = {}
data['failures'] = []
data['totals'] = []
for day, fail, total in query_results:
dates.append(day)
timestamp = calendar.timegm(day.timetuple()) * 1000
data['failures'].append((timestamp, int(fail)))
data['totals'].append((timestamp, int(total)))
data_platforms[platform] = {'data': data, 'dates': get_date_range(dates)}
session.close()
return data_platforms
def run_results_day_flot_query():
"""
This function returns the total failures/total jobs data per day for all platforms.
It is sending the data in the format required by flot.Flot is a jQuery package used
for 'attractive' plotting
"""
start_date, end_date = clean_date_params(request.args)
platforms = ['android4.0',
'android2.3',
'linux32',
'winxp',
'win7',
'win8',
'osx10.6',
'osx10.7',
'osx10.8']
data_platforms = {}
for platform in platforms:
query_results = session.query(Testjobs.date.label('day'),
func.count(Testjobs.result == 'testfailed'
).label("failures"),
func.count(Testjobs).label('totals')).filter(
and_(Testjobs.platform == platform,
Testjobs.date >= start_date, Testjobs.date <= end_date)).group_by('day').all()
dates = []
data = {}
data['failures'] = []
data['totals'] = []
for day, fail, total in query_results:
dates.append(day)
timestamp = calendar.timegm(day.timetuple()) * 1000
data['failures'].append((timestamp, int(fail)))
data['totals'].append((timestamp, int(total)))
data_platforms[platform] = {'data': data, 'dates': get_date_range(dates)}
session.close()
return data_platforms
def __init__(self):
self.email = input("Enter your email id: ")
self.password = input("Enter your Password: "******"*****@*****.**" and self.password == "Mohit#24":
print("Welcome Admin")
AdminActions.admin_action()
else:
print("Welcome Customer")
try:
# Check customer account exist or not
all_email_password = session.query(Customer).filter(
and_(Customer.email_id == self.email,
Customer.password == self.password)).one()
if all_email_password:
CustomerActions.customer_action()
except NoResultFound:
print(
"Account is not available for this email id. Please create your account! :)"
)
User.add_user()
CustomerActions.customer_action()
def reset_preseed():
data = session.query(JobPriorities.expires, JobPriorities.id)\
.filter(JobPriorities.expires != None).all()
now = datetime.datetime.now()
for item in data:
try:
dv = datetime.datetime.strptime(item[0], "%Y-%M-%d")
except ValueError:
# TODO: consider updating column to have expires=None?
continue
except TypeError:
dv = datetime.datetime.combine(item[0].today(),
datetime.datetime.min.time())
# reset expire field if date is today or in the past
if dv.date() <= now.date():
conn = engine.connect()
statement = update(JobPriorities)\
.where(JobPriorities.id == item[1])\
.values(expires=None)
conn.execute(statement)
def clear_expiration_field_for_expired_jobs():
data = session.query(JobPriorities.expires, JobPriorities.id).filter(
JobPriorities.expires != None).all() # flake8: noqa
now = datetime.datetime.now()
for item in data:
try:
expiration_date = datetime.datetime.strptime(item[0], "%Y-%M-%d")
except ValueError:
# TODO: consider updating column to have expires=None?
LOG.warning('Failed to downcast to datetime for ({},{})'.format(
item[1], item[0]))
continue
except TypeError:
expiration_date = datetime.datetime.combine(
item[0].today(), datetime.datetime.min.time())
# reset expiration field if the date is today or in the past
if expiration_date.date() <= now.date():
conn = engine.connect()
statement = update(JobPriorities)\
.where(JobPriorities.id == item[1])\
.values(expires=None)
conn.execute(statement)
def _update_job_priority_table(data):
"""Add new jobs to the priority table and update the build system if required."""
LOG.info('Fetch all rows from the job priority table.')
# Get all rows of job priorities
db_data = session.query(JobPriorities.id, JobPriorities.testtype,
JobPriorities.buildtype, JobPriorities.platform,
JobPriorities.priority, JobPriorities.timeout,
JobPriorities.expires,
JobPriorities.buildsystem).all()
# TODO: write test for this
# When the table is empty it means that we're starting the system for the first time
# and we're going to use different default values
map = {}
if not len(db_data) == 0:
priority = 1
timeout = 0
# Using %Y-%m-%d fixes this issue:
# Warning: Incorrect date value: '2016-10-28 17:36:58.153265' for column 'expires' at row 1
expiration_date = (datetime.datetime.now() +
datetime.timedelta(days=14)).strftime("%Y-%m-%d")
# Creating this data structure which will reduce how many times we iterate through the DB rows
for row in db_data:
key = tuple(row[1:4])
# This is guaranteed by a unique composite index for these 3 fields in models.py
assert key not in map,\
'"{}" should be a unique row and that is unexpected.'.format(key)
# (testtype, buildtype, platform)
map[key] = {'pk': row[0], 'build_system_type': row[7]}
else:
priority = 5
timeout = 5400
expiration_date = None
total_jobs = len(data)
new_jobs = 0
failed_changes = 0
updated_jobs = 0
# Loop through sanitized jobs, add new jobs and update the build system if needed
for job in data:
_buildsystem = job["build_system_type"]
key = _unique_key(job)
if key in map:
# We already know about this job, we might need to update the build system
row_build_system_type = map[key]['build_system_type']
if row_build_system_type == '*' or _buildsystem == '*':
# We don't need to update anything
pass
else:
# We're seeing the job again but for another build system (e.g. buildbot vs
# taskcluster). We need to change it to '*'
if row_build_system_type != _buildsystem:
_buildsystem = "*"
# Update table with new buildsystem
try:
conn = engine.connect()
statement = update(JobPriorities).where(
JobPriorities.id == map[key]['pk_key']).values(
buildsystem=_buildsystem)
conn.execute(statement)
LOG.info('Updated {}/{} from {} to {}'.format(
job['testtype'], job['platform_option'],
job['build_system_type'], _buildsystem))
updated_jobs += 1
except Exception as e:
LOG.info("key = %s, buildsystem = %s" %
(key, _buildsystem))
LOG.info("exception updating jobPriorities: %s" % e)
else:
# We have a new job from runnablejobs to add to our master list
try:
jobpriority = JobPriorities(str(job["testtype"]),
str(job["platform_option"]),
str(job["platform"]), priority,
timeout, expiration_date,
_buildsystem)
session.add(jobpriority)
session.commit()
LOG.info('New job was found ({},{},{},{})'.format(
job['testtype'],
job['platform_option'],
job['platform'],
_buildsystem,
))
new_jobs += 1
except Exception as error:
session.rollback()
LOG.warning(error)
failed_changes += 1
finally:
session.close()
LOG.info(
'We have {} new jobs and {} updated jobs out of {} total jobs processed.'
.format(new_jobs, updated_jobs, total_jobs))
if failed_changes != 0:
LOG.error(
'We have failed {} changes out of {} total jobs processed.'.format(
failed_changes, total_jobs))
def update_preseed():
"""
We sync preseed.json to jobpririties in server on startup, since that is
the only time we expect preseed.json to change.
"""
# get preseed data first
preseed_path = os.path.join(os.path.dirname(SCRIPT_DIR), 'src', 'preseed.json')
preseed = []
with open(preseed_path, 'r') as fHandle:
preseed = json.load(fHandle)
# Preseed data will have fields: buildtype,testtype,platform,priority,timeout,expires
# The expires field defaults to 2 weeks on a new job in the database
# Expires field has a date "YYYY-MM-DD", but can have "*" to indicate never
# Typical priority will be 1, but if we want to force coalescing we can do that
# One hack is that if we have a * in a buildtype,testtype,platform field, then
# we assume it is for all flavors of the * field: i.e. linux64,pgo,* - all tests
# assumption - preseed fields are sanitized already - move parse_testtype to utils.py ?
for job in preseed:
data = session.query(JobPriorities.id,
JobPriorities.testtype,
JobPriorities.buildtype,
JobPriorities.platform,
JobPriorities.priority,
JobPriorities.timeout,
JobPriorities.expires,
JobPriorities.buildsystem)
if job['testtype'] != '*':
data = data.filter(getattr(JobPriorities, 'testtype') == job['testtype'])
if job['buildtype'] != '*':
data = data.filter(getattr(JobPriorities, 'buildtype') == job['buildtype'])
if job['platform'] != '*':
data = data.filter(getattr(JobPriorities, 'platform') == job['platform'])
data = data.all()
_buildsystem = job["build_system_type"]
# TODO: edge case: we add future jobs with a wildcard, when jobs show up
# remove the wildcard, apply priority/timeout/expires to new jobs
# Deal with the case where we have a new entry in preseed
if len(data) == 0:
_expires = job['expires']
if _expires == '*':
_expires = str(datetime.now().date() + timedelta(days=365))
LOG.info("adding a new unknown job to the database: %s" % job)
newjob = JobPriorities(job['testtype'],
job['buildtype'],
job['platform'],
job['priority'],
job['timeout'],
_expires,
_buildsystem)
session.add(newjob)
session.commit()
session.close()
continue
# We can have wildcards, so loop on all returned values in data
for d in data:
changed = False
LOG.info("updating existing job %s/%s/%s" % (d[1], d[2], d[3]))
_expires = job['expires']
_priority = job['priority']
_timeout = job['timeout']
# we have a taskcluster job in the db, and new job in preseed
if d[7] != _buildsystem:
_buildsystem = "*"
changed = True
# When we have a defined date to expire a job, parse and use it
if _expires == '*':
_expires = str(datetime.now().date() + timedelta(days=365))
try:
dv = datetime.strptime(_expires, "%Y-%M-%d").date()
except ValueError:
continue
# When we have expired, use existing priority/timeout, reset expires
if dv <= datetime.now().date():
LOG.info(" -- past the expiration date- reset!")
_expires = ''
_priority = d[4]
_timeout = d[5]
changed = True
if changed:
# TODO: do we need to try/except/finally with commit/rollback statements
conn = engine.connect()
statement = update(JobPriorities).where(
JobPriorities.id == d[0]).values(
priority=_priority,
timeout=_timeout,
expires=_expires,
buildsystem=_buildsystem)
conn.execute(statement)
def run_seta_details_query():
startDate, date = clean_date_params(request.args)
active = sanitize_bool(request.args.get("active", 0))
buildbot = sanitize_bool(request.args.get("buildbot", 0))
branch = sanitize_string(request.args.get("branch", ''))
taskcluster = sanitize_bool(request.args.get("taskcluster", 0))
priority = sanitize_string(request.args.get("priority", "low"))
jobnames = JOBSDATA.jobnames_query()
if date == "" or date == "latest":
today = datetime.now()
date = today.strftime("%Y-%m-%d")
date = "%s" % date
query = session.query(Seta.jobtype).filter(Seta.date == date).all()
retVal = {}
retVal[date] = []
jobtype = []
# we only support fx-team and mozilla-inbound branch in seta
if (str(branch) in ['fx-team', 'mozilla-inbound', 'autoland']) is not True \
and str(branch) != '':
abort(404)
for d in query:
parts = d[0].split("'")
jobtype.append([parts[1], parts[3], parts[5]])
alljobs = JOBSDATA.jobtype_query()
# Because we store high value jobs in seta table as default,
# so we return low value jobs(default) when the priority is 'low',
# otherwise we return high value jobs.
if priority == 'low':
low_value_jobs = [low_value_job for low_value_job in alljobs if
low_value_job not in jobtype]
jobtype = low_value_jobs
if active:
active_jobs = []
for job in alljobs:
found = False
for j in jobtype:
if j[0] == job[0] and j[1] == job[1] and j[2] == job[2]:
found = True
break
if not found:
active_jobs.append(job)
jobtype = active_jobs
if buildbot:
active_jobs = []
# pick up buildbot jobs from job list to faster the filter process
buildbot_jobs = [job for job in jobnames if job['buildplatform'] == 'buildbot']
# find out the correspond job detail information
for job in jobtype:
for j in buildbot_jobs:
if j['name'] == job[2] and j['platform'] == job[0] and j['buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'] if branch is 'mozilla-inbound'
else j['ref_data_name'].replace('mozilla-inbound', branch))
jobtype = active_jobs
if taskcluster:
active_jobs = []
taskcluster_jobs = [job for job in jobnames if job['buildplatform'] == 'taskcluster']
for job in jobtype:
for j in taskcluster_jobs:
if j['name'] == job[2] and j['platform'] == job[0] and j['buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'])
jobtype = active_jobs
retVal[date] = jobtype
return {"jobtypes": retVal}
def add_user():
""" Add new user/customer """
name = input("Enter your name: ")
email_id = input("Enter email id: ")
# Validate Email id
while True:
if not validate_email(email_id):
print(
"Entered Email id is invalid. Please enter valid email id")
email_id = input("Enter email id: ")
else:
try:
email_id_object = session.query(Customer).filter(
Customer.email_id == email_id).one()
if email_id_object:
print("Email id already exist.")
email_id = input("Enter email id: ")
except NoResultFound:
break
# TODO: Use getpass module to take hidden password
password = input("Enter password: "******"Enter password: "******"Enter confirm password: "******"Password and confirm password is not same. Please try again")
password = input("Enter password: "******"Enter confirm password: "******"Enter your mobile number: ")
# Validate mobile number
while True:
if not len(mobile_number) == 10:
print(
"Entered mobile number is invalid. Please enter 10 digits mobile number"
)
mobile_number = input("Enter mobile number: ")
else:
try:
mobile_no_object = session.query(Customer).filter(
Customer.mobile_number == mobile_number).one()
if mobile_no_object:
print("Mobile number already exist")
mobile_number = input("Enter your mobile number: ")
except NoResultFound:
break
c = Customer()
c.add_customer(name, email_id, password, mobile_number)
session.add(c)
session.commit()
session.close()
def migration(args):
limit = int(args.limit)
startDate = args.startDate
offset = 0
url = URL % (startDate, limit, offset)
try:
response = retry(requests.get, args=(url, )).json()
except Exception as error:
# we will return an empty 'result' list if got exception here
logger.debug("the request to %s failed, due to %s" % (url, error))
response = {'result': []}
datasets = response['result']
session.query(Testjobs).filter(
Testjobs.date >= '%s 00:00:00' % startDate).delete()
while len(datasets) > 0:
for data in datasets:
testtype = data['testtype']
# spidermonkey builds
if len(testtype.split('pider')) > 1:
continue
# docker generation jobs
if len(testtype.split('ocker')) > 1:
continue
# hazzard builds
if len(testtype.split('az')) > 1:
continue
# decision tasks
if len(testtype.split('ecision')) > 1:
continue
# TODO: figure out a better naming strategy here
# builds, linting, etc.
if testtype.startswith('[TC]'):
continue
# skip talos jobs
if testtype in [
'chromez', 'tp5o', 'g1', 'g2', 'g3', 'g4', 'xperf',
'chromez-e10s', 'tp5o-e10s', 'g1-e10s', 'g2-e10s',
'g3-e10s', 'g4-e10s', 'xperf-e10s', 'dromaeojs',
'dromaeojs-e10s', 'svgr', 'svgr-e10s', 'remote-tsvgx',
'remote-tp4m_chrome', 'other', 'other-e10s'
]:
continue
# linter jobs
if testtype in [
'Static Analysis Opt', 'ESLint', 'Android lint', 'lint'
]:
continue
# builds
if testtype in [
'nightly', 'Android armv7 API 15+', 'ASan Dbg', 'valgrind',
'Android armv7 API 15+ Dbg', 'ASan Debug', 'pgo-build',
'ASan Opt', 'build'
]:
continue
# hidden/lower tier tests, not sure of CI system, old jobs
if testtype in [
'media-youtube-tests', 'external-media-tests',
'luciddream', 'media-tests'
]:
continue
Testjob = Testjobs(data['slave'], data['result'],
data['build_system_type'], data['duration'],
data['platform'], data['buildtype'], testtype,
data['bugid'], data['branch'], data['revision'],
data['date'], data['failure_classification'],
data['failures'])
try:
session.add(Testjob)
session.commit()
except Exception as error:
logging.warning(error)
session.rollback()
finally:
session.close()
# The process will move forward by set offset
offset += limit
url = URL % (startDate, limit, offset)
response = retry(requests.get, args=(url, )).json()
datasets = response['result']
def run_platform_query():
platform = sanitize_string(request.args.get("platform"))
build_system_type = sanitize_string(request.args.get("build_system_type"))
start_date, end_date = clean_date_params(request.args)
log_message = 'platform: %s startDate: %s endDate: %s' % (
platform, start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'))
app.logger.debug(log_message)
csets = session.query(Testjobs.revision).distinct().\
filter(and_(Testjobs.platform == platform,
Testjobs.branch == 'mozilla-central',
Testjobs.date.between(start_date, end_date),
Testjobs.build_system_type == build_system_type)).order_by(desc(Testjobs.date))
cset_summaries = []
test_summaries = {}
dates = []
labels = 'green orange blue red'.split()
summary = {result: 0 for result in labels}
for cset in csets:
cset_id = cset[0]
cset_summary = CSetSummary(cset_id)
test_results = session.query(Testjobs.result, Testjobs.testtype, Testjobs.date).\
filter(and_(Testjobs.platform == platform,
Testjobs.buildtype == 'opt',
Testjobs.revision == cset_id,
Testjobs.build_system_type == build_system_type)).all().order_by(
Testjobs.testtype)
for res, testtype, date in test_results:
test_summary = test_summaries.setdefault(testtype, summary.copy())
if res == 'success':
cset_summary.green[testtype] += 1
test_summary['green'] += 1
elif res == 'testfailed':
cset_summary.orange[testtype] += 1
test_summary['orange'] += 1
elif res == 'retry':
cset_summary.blue[testtype] += 1
test_summary['blue'] += 1
elif res == 'exception' or res == 'busted':
cset_summary.red[testtype] += 1
test_summary['red'] += 1
elif res == 'usercancel':
app.logger.debug('usercancel')
else:
app.logger.debug('UNRECOGNIZED RESULT: %s' % res)
dates.append(date)
cset_summaries.append(cset_summary)
# sort tests alphabetically and append total & percentage to end of the list
test_types = sorted(test_summaries.keys())
test_types += ['total', 'percentage']
# calculate total stats and percentage
total = Counter()
percentage = {}
for test in test_summaries:
total.update(test_summaries[test])
test_count = sum(total.values())
for key in total:
percentage[key] = round((100.0 * total[key] / test_count), 2)
fail_rates = calculate_fail_rate(passes=total['green'],
retries=total['blue'],
totals=test_count)
test_summaries['total'] = total
test_summaries['percentage'] = percentage
session.close()
return {
'testTypes': test_types,
'byRevision': cset_summaries,
'byTest': test_summaries,
'failRates': fail_rates,
'dates': get_date_range(dates)
}
def run_seta_details_query():
# TODO: remove inactive when buildbot api queries s/inactive/priority/
inactive = sanitize_bool(request.args.get("inactive", 0))
buildbot = sanitize_bool(request.args.get("buildbot", 0))
branch = sanitize_string(request.args.get("branch", ''))
taskcluster = sanitize_bool(request.args.get("taskcluster", 0))
priority = int(sanitize_string(request.args.get("priority", '1')))
jobnames = JOBSDATA.jobnames_query()
date = str(datetime.now().date())
if inactive == 1:
priority = 5
else:
priority = 1
# TODO: we can make this a variable priority in the future based on input
query = session.query(
JobPriorities.platform, JobPriorities.buildtype,
JobPriorities.testtype).filter(JobPriorities.priority == 1).all()
retVal = {}
retVal[date] = []
jobtype = []
# we only support fx-team, autoland, and mozilla-inbound branch in seta
if (str(branch) in ['fx-team', 'mozilla-inbound', 'autoland']) is not True \
and str(branch) != '':
abort(404)
for d in query:
jobtype.append([d[0], d[1], d[2]])
# We call valve to determine what kind of jobs we should return only if
# this request is comes from taskcluster. Otherwise, we just return what people
# request for.
if request.headers.get('User-Agent', '') == 'TaskCluster':
# We should return full job list as a fallback, if it's a request from
# taskcluster and without head_rev or pushlog_id in there
if head_rev or pushlog_id:
priority = valve(head_rev, pushlog_id, branch, priority)
else:
priority = 0
alljobs = JOBSDATA.jobtype_query()
# Because we store high value jobs in seta table as default,
# so we return low value jobs, means no failure related with this job as default
# priority = 0; run all the jobs
if priority == 0:
jobtype = alljobs
# priority =5 run all low value jobs
elif priority == 5:
low_value_jobs = [
low_value_job for low_value_job in alljobs
if low_value_job not in jobtype
]
jobtype = low_value_jobs
# priority =1, run all high value jobs
elif priority == 1:
pass # use jobtype as a high value query
# TODO: filter out based on buildsystem from database, either 'buildbot' or '*'
if buildbot:
active_jobs = []
# pick up buildbot jobs from job list to faster the filter process
buildbot_jobs = [
job for job in jobnames if job['buildplatform'] == 'buildbot'
]
# find out the correspond job detail information
for job in jobtype:
for j in buildbot_jobs:
if j['name'] == job[2] and j['platform'] == job[0] and j[
'buildtype'] == job[1]:
active_jobs.append(
j['ref_data_name'] if branch is 'mozilla-inbound' else
j['ref_data_name'].replace('mozilla-inbound', branch))
jobtype = active_jobs
# TODO: filter out based on buildsystem from database, either 'taskcluster' or '*'
if taskcluster:
active_jobs = []
taskcluster_jobs = [
job for job in jobnames if job['buildplatform'] == 'taskcluster'
]
for job in jobtype:
for j in taskcluster_jobs:
if j['name'] == job[2] and j['platform'] == job[0] and j[
'buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'])
jobtype = active_jobs
retVal[date] = jobtype
return {"jobtypes": retVal}
def update_preseed():
"""
We sync preseed.json to jobpririties in server on startup, since that is
the only time we expect preseed.json to change.
"""
# get preseed data first
preseed_path = os.path.join(os.path.dirname(SCRIPT_DIR), 'src',
'preseed.json')
preseed = []
with open(preseed_path, 'r') as fHandle:
preseed = json.load(fHandle)
# Preseed data will have fields: buildtype,testtype,platform,priority,timeout,expires
# The expires field defaults to 2 weeks on a new job in the database
# Expires field has a date "YYYY-MM-DD", but can have "*" to indicate never
# Typical priority will be 1, but if we want to force coalescing we can do that
# One hack is that if we have a * in a buildtype,testtype,platform field, then
# we assume it is for all flavors of the * field: i.e. linux64,pgo,* - all tests
# assumption - preseed fields are sanitized already - move parse_testtype to utils.py ?
for job in preseed:
data = session.query(JobPriorities.id, JobPriorities.testtype,
JobPriorities.buildtype, JobPriorities.platform,
JobPriorities.priority, JobPriorities.timeout,
JobPriorities.expires, JobPriorities.buildsystem)
if job['testtype'] != '*':
data = data.filter(
getattr(JobPriorities, 'testtype') == job['testtype'])
if job['buildtype'] != '*':
data = data.filter(
getattr(JobPriorities, 'buildtype') == job['buildtype'])
if job['platform'] != '*':
data = data.filter(
getattr(JobPriorities, 'platform') == job['platform'])
data = data.all()
_buildsystem = job["build_system_type"]
# TODO: edge case: we add future jobs with a wildcard, when jobs show up
# remove the wildcard, apply priority/timeout/expires to new jobs
# Deal with the case where we have a new entry in preseed
if len(data) == 0:
_expires = job['expires']
if _expires == '*':
_expires = str(datetime.now().date() + timedelta(days=365))
LOG.info("adding a new unknown job to the database: %s" % job)
newjob = JobPriorities(job['testtype'], job['buildtype'],
job['platform'], job['priority'],
job['timeout'], _expires, _buildsystem)
session.add(newjob)
session.commit()
session.close()
continue
# We can have wildcards, so loop on all returned values in data
for d in data:
changed = False
LOG.info("updating existing job %s/%s/%s" % (d[1], d[2], d[3]))
_expires = job['expires']
_priority = job['priority']
_timeout = job['timeout']
# we have a taskcluster job in the db, and new job in preseed
if d[7] != _buildsystem:
_buildsystem = "*"
changed = True
# When we have a defined date to expire a job, parse and use it
if _expires == '*':
_expires = str(datetime.now().date() + timedelta(days=365))
try:
dv = datetime.strptime(_expires, "%Y-%M-%d").date()
except ValueError:
continue
# When we have expired, use existing priority/timeout, reset expires
if dv <= datetime.now().date():
LOG.info(" -- past the expiration date- reset!")
_expires = ''
_priority = d[4]
_timeout = d[5]
changed = True
if changed:
# TODO: do we need to try/except/finally with commit/rollback statements
conn = engine.connect()
statement = update(JobPriorities).where(
JobPriorities.id == d[0]).values(priority=_priority,
timeout=_timeout,
expires=_expires,
buildsystem=_buildsystem)
conn.execute(statement)
def run_platform_query():
platform = sanitize_string(request.args.get("platform"))
build_system_type = sanitize_string(request.args.get("build_system_type"))
start_date, end_date = clean_date_params(request.args)
log_message = 'platform: %s startDate: %s endDate: %s' % (platform,
start_date.strftime('%Y-%m-%d'),
end_date.strftime('%Y-%m-%d'))
app.logger.debug(log_message)
csets = session.query(Testjobs.revision).distinct().\
filter(and_(Testjobs.platform == platform,
Testjobs.branch == 'mozilla-central',
Testjobs.date.between(start_date, end_date),
Testjobs.build_system_type == build_system_type)).order_by(desc(Testjobs.date))
cset_summaries = []
test_summaries = {}
dates = []
labels = 'green orange blue red'.split()
summary = {result: 0 for result in labels}
for cset in csets:
cset_id = cset[0]
cset_summary = CSetSummary(cset_id)
test_results = session.query(Testjobs.result, Testjobs.testtype, Testjobs.date).\
filter(and_(Testjobs.platform == platform,
Testjobs.buildtype == 'opt',
Testjobs.revision == cset_id,
Testjobs.build_system_type == build_system_type)).all().order_by(
Testjobs.testtype)
for res, testtype, date in test_results:
test_summary = test_summaries.setdefault(testtype, summary.copy())
if res == 'success':
cset_summary.green[testtype] += 1
test_summary['green'] += 1
elif res == 'testfailed':
cset_summary.orange[testtype] += 1
test_summary['orange'] += 1
elif res == 'retry':
cset_summary.blue[testtype] += 1
test_summary['blue'] += 1
elif res == 'exception' or res == 'busted':
cset_summary.red[testtype] += 1
test_summary['red'] += 1
elif res == 'usercancel':
app.logger.debug('usercancel')
else:
app.logger.debug('UNRECOGNIZED RESULT: %s' % res)
dates.append(date)
cset_summaries.append(cset_summary)
# sort tests alphabetically and append total & percentage to end of the list
test_types = sorted(test_summaries.keys())
test_types += ['total', 'percentage']
# calculate total stats and percentage
total = Counter()
percentage = {}
for test in test_summaries:
total.update(test_summaries[test])
test_count = sum(total.values())
for key in total:
percentage[key] = round((100.0 * total[key] / test_count), 2)
fail_rates = calculate_fail_rate(passes=total['green'],
retries=total['blue'],
totals=test_count)
test_summaries['total'] = total
test_summaries['percentage'] = percentage
session.close()
return {'testTypes': test_types,
'byRevision': cset_summaries,
'byTest': test_summaries,
'failRates': fail_rates,
'dates': get_date_range(dates)}
def run_seta_details_query():
buildbot = sanitize_bool(request.args.get("buildbot", 0))
branch = sanitize_string(request.args.get("branch", 'mozilla-inbound'))
taskcluster = sanitize_bool(request.args.get("taskcluster", 0))
priority = int(sanitize_string(request.args.get("priority", '5')))
jobnames = JOBSDATA.jobnames_query()
date = str(datetime.now().date())
retVal = {}
retVal[date] = []
jobtype = []
# we only support fx-team, autoland, and mozilla-inbound branch in seta
if (str(branch) in ['fx-team', 'mozilla-inbound', 'autoland']) is not True \
and str(branch) != '':
abort(404)
# For the case of TaskCluster request, we don't care which priority the user request.
# We return jobs depend on the strategy that we return high value jobs as default and
# return all jobs for every 5 push or 90 minutes for that branch.
if request.headers.get('User-Agent', '') == 'TaskCluster':
# we make taskcluster to 1 if it's a request from taskcluster, it's more reasonable and
# can simplify the request url.
taskcluster = 1
# we will return all jobs for every 90 minutes, so the return_all_jobs flag will been
# set to true if the time limit been reached.
return_all_jobs = False
# We should return full job list as a fallback, if it's a request from
# taskcluster and without head_rev or pushlog_id in there
try:
branch_info = session.query(TaskRequests.counter,
TaskRequests.datetime,
TaskRequests.reset_delta).filter(
TaskRequests.branch == branch).all()
except:
branch_info = []
time_of_now = datetime.now()
# If we got nothing related with that branch, we should create it.
if len(branch_info) == 0:
# time_of_lastreset is not a good name anyway :(
# And we treat all branches' reset_delta is 90 seconds, we should find a
# better delta for them in the further.
branch_data = TaskRequests(str(branch), 1, time_of_now, RESET_DELTA)
try:
session.add(branch_data)
session.commit()
except Exception as error:
LOG.debug(error)
session.rollback()
finally:
session.close()
counter = 1
time_string = time_of_now
reset_delta = RESET_DELTA
# We should update it if that branch had already been stored.
else:
counter, time_string, reset_delta = branch_info[0]
counter += 1
conn = engine.connect()
statement = update(TaskRequests).where(
TaskRequests.branch == branch).values(
counter=counter)
conn.execute(statement)
delta = (time_of_now - time_string).total_seconds()
# we should update the time recorder if the elapse time had
# reach the time limit of that branch.
if delta >= reset_delta:
conn = engine.connect()
statement = update(TaskRequests).where(
TaskRequests.branch == branch).values(
datetime=time_of_now)
conn.execute(statement)
# we need to set the return_all_jobs flag to true.
return_all_jobs = True
# we query all jobs rather than jobs filter by the requested priority in here,
# Because we need to set the job returning strategy depend on different job priority.
query = session.query(JobPriorities.platform,
JobPriorities.buildtype,
JobPriorities.testtype,
JobPriorities.priority,
JobPriorities.timeout
).all()
for d in query:
# we only return that job if it hasn't reach the timeout limit. And the
# timeout is zero means this job need always running.
if delta < d[4] or d[4] == 0:
# Due to the priority of all high value jobs is 1, and we
# need to return all jobs for every 5 pushes(for now).
if counter % d[3] != 0:
jobtype.append([d[0], d[1], d[2]])
# we need to return all jobs for every 90 minutes, so all jobs will been returned
# if the delta is larger than 5400
elif return_all_jobs:
jobtype.append([d[0], d[1], d[2]])
# We don't care about the timeout variable of job if it's not a taskcluster request.
else:
query = session.query(JobPriorities.platform,
JobPriorities.buildtype,
JobPriorities.testtype,
JobPriorities.priority,
).all()
# priority = 0; run all the jobs
if priority != 1 and priority != 5:
priority = 0
# Because we store high value jobs in seta table as default,
# so we return low value jobs, means no failure related with this job as default
if priority == 0:
jobtype = JOBSDATA.jobtype_query() # All jobs regardless of priority
# priority =5 run all low value jobs
else:
joblist = [job for job in query if job[3] == priority]
for j in joblist:
jobtype.append([j[0], j[1], j[2]])
# TODO: filter out based on buildsystem from database, either 'buildbot' or '*'
if buildbot:
active_jobs = []
# pick up buildbot jobs from job list to faster the filter process
buildbot_jobs = [job for job in jobnames if job['buildplatform'] == 'buildbot']
# find out the correspond job detail information
for job in jobtype:
for j in buildbot_jobs:
if j['name'] == job[2] and j['platform'] == job[0] and j['buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'] if branch is 'mozilla-inbound'
else j['ref_data_name'].replace(
'mozilla-inbound', branch))
jobtype = active_jobs
# TODO: filter out based on buildsystem from database, either 'taskcluster' or '*'
if taskcluster:
active_jobs = []
taskcluster_jobs = [job for job in jobnames if job['buildplatform'] == 'taskcluster']
for job in jobtype:
# we need to retranslate the jobtype back to the proper data form after all.
job[2] = job[2].replace('e10s-browser-chrome', 'browser-chrome-e10s')
job[2] = job[2].replace('e10s-devtools-chrome', 'devtools-chrome-e10s')
job[2] = job[2].replace('gl-', 'webgl-')
for j in taskcluster_jobs:
if job[2] in j['name'] and j['platform'] == job[0] and j['buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'])
jobtype = active_jobs
retVal[date] = jobtype
return {"jobtypes": retVal}
def _update_job_priority_table(data):
"""Add new jobs to the priority table and update the build system if required."""
LOG.info('Fetch all rows from the job priority table.')
# Get all rows of job priorities
db_data = session.query(JobPriorities.id,
JobPriorities.testtype,
JobPriorities.buildtype,
JobPriorities.platform,
JobPriorities.priority,
JobPriorities.timeout,
JobPriorities.expires,
JobPriorities.buildsystem).all()
# TODO: write test for this
# When the table is empty it means that we're starting the system for the first time
# and we're going to use different default values
map = {}
if not len(db_data) == 0:
priority = 1
timeout = 0
# Using %Y-%m-%d fixes this issue:
# Warning: Incorrect date value: '2016-10-28 17:36:58.153265' for column 'expires' at row 1
expiration_date = (datetime.datetime.now() + datetime.timedelta(days=14)).strftime("%Y-%m-%d")
# Creating this data structure which will reduce how many times we iterate through the DB rows
for row in db_data:
key = tuple(row[1:4])
# This is guaranteed by a unique composite index for these 3 fields in models.py
assert key not in map,\
'"{}" should be a unique row and that is unexpected.'.format(key)
# (testtype, buildtype, platform)
map[key] = {'pk': row[0], 'build_system_type': row[7]}
else:
priority = 5
timeout = 5400
expiration_date = None
total_jobs = len(data)
new_jobs = 0
failed_changes = 0
updated_jobs = 0
# Loop through sanitized jobs, add new jobs and update the build system if needed
for job in data:
_buildsystem = job["build_system_type"]
key = _unique_key(job)
if key in map:
# We already know about this job, we might need to update the build system
row_build_system_type = map[key]['build_system_type']
if row_build_system_type == '*' or _buildsystem == '*':
# We don't need to update anything
pass
else:
# We're seeing the job again but for another build system (e.g. buildbot vs
# taskcluster). We need to change it to '*'
if row_build_system_type != _buildsystem:
_buildsystem = "*"
# Update table with new buildsystem
try:
conn = engine.connect()
statement = update(JobPriorities).where(
JobPriorities.id == map[key]['pk_key']).values(buildsystem=_buildsystem)
conn.execute(statement)
LOG.info('Updated {}/{} from {} to {}'.format(
job['testtype'], job['platform_option'],
job['build_system_type'], _buildsystem
))
updated_jobs += 1
except Exception as e:
LOG.info("key = %s, buildsystem = %s" % (key, _buildsystem))
LOG.info("exception updating jobPriorities: %s" % e)
else:
# We have a new job from runnablejobs to add to our master list
try:
jobpriority = JobPriorities(
str(job["testtype"]),
str(job["platform_option"]),
str(job["platform"]),
priority,
timeout,
expiration_date,
_buildsystem
)
session.add(jobpriority)
session.commit()
LOG.info('New job was found ({},{},{},{})'.format(
job['testtype'], job['platform_option'], job['platform'], _buildsystem,))
new_jobs += 1
except Exception as error:
session.rollback()
LOG.warning(error)
failed_changes += 1
finally:
session.close()
LOG.info('We have {} new jobs and {} updated jobs out of {} total jobs processed.'.format(
new_jobs, updated_jobs, total_jobs
))
if failed_changes != 0:
LOG.error('We have failed {} changes out of {} total jobs processed.'.format(
failed_changes, total_jobs
))
def prepare_the_database():
# wipe up the job data older than 90 days
date = (datetime.datetime.now() -
datetime.timedelta(days=SETA_WINDOW)).strftime('%Y-%m-%d')
session.query(Seta).filter(Seta.date <= date)
def run_slaves_query():
start_date, end_date = clean_date_params(request.args)
days_to_show = (end_date - start_date).days
if days_to_show <= 8:
jobs = 5
else:
jobs = int(round(days_to_show * 0.4))
info = 'Only slaves with more than %d jobs are displayed.' % jobs
query_results = session.query(
Testjobs.slave, Testjobs.result, Testjobs.date).filter(
and_(
Testjobs.result.in_(
["retry", "testfailed", "success", "busted", "exception"]),
Testjobs.date.between(start_date,
end_date))).all().order_by(Testjobs.date)
session.close()
if not query_results:
return
data = {}
labels = 'fail retry infra success total'.split()
summary = {result: 0 for result in labels}
summary['jobs_since_last_success'] = 0
dates = []
for name, result, date in query_results:
data.setdefault(name, summary.copy())
data[name]['jobs_since_last_success'] += 1
if result == 'testfailed':
data[name]['fail'] += 1
elif result == 'retry':
data[name]['retry'] += 1
elif result == 'success':
data[name]['success'] += 1
data[name]['jobs_since_last_success'] = 0
elif result == 'busted' or result == 'exception':
data[name]['infra'] += 1
data[name]['total'] += 1
dates.append(date)
# filter slaves
slave_list = [slave for slave in data if data[slave]['total'] > jobs]
# calculate failure rate only for slaves that we're going to display
for slave in slave_list:
results = data[slave]
fail_rates = calculate_fail_rate(results['success'], results['retry'],
results['total'])
data[slave]['sfr'] = fail_rates
platforms = {}
# group slaves by platform and calculate platform failure rate
slaves = sorted(data.keys())
for platform, slave_group in groupby(slaves,
lambda x: x.rsplit('-', 1)[0]):
slaves = list(slave_group)
# don't calculate failure rate for platform we're not going to show
if not any(slave in slaves for slave in slave_list):
continue
platforms[platform] = {}
results = {}
for label in ['success', 'retry', 'total']:
r = reduce(lambda x, y: x + y,
[data[slave][label] for slave in slaves])
results[label] = r
fail_rates = calculate_fail_rate(results['success'], results['retry'],
results['total'])
platforms[platform].update(fail_rates)
# remove data that we don't need
for slave in data.keys():
if slave not in slave_list:
del data[slave]
return {
'slaves': data,
'platforms': platforms,
'dates': get_date_range(dates),
'disclaimer': info
}
def run_slaves_query():
start_date, end_date = clean_date_params(request.args)
days_to_show = (end_date - start_date).days
if days_to_show <= 8:
jobs = 5
else:
jobs = int(round(days_to_show * 0.4))
info = 'Only slaves with more than %d jobs are displayed.' % jobs
query_results = session.query(Testjobs.slave, Testjobs.result, Testjobs.date).filter(
and_(Testjobs.result.in_(["retry", "testfailed", "success", "busted", "exception"]),
Testjobs.date.between(start_date, end_date))).all().order_by(Testjobs.date)
session.close()
if not query_results:
return
data = {}
labels = 'fail retry infra success total'.split()
summary = {result: 0 for result in labels}
summary['jobs_since_last_success'] = 0
dates = []
for name, result, date in query_results:
data.setdefault(name, summary.copy())
data[name]['jobs_since_last_success'] += 1
if result == 'testfailed':
data[name]['fail'] += 1
elif result == 'retry':
data[name]['retry'] += 1
elif result == 'success':
data[name]['success'] += 1
data[name]['jobs_since_last_success'] = 0
elif result == 'busted' or result == 'exception':
data[name]['infra'] += 1
data[name]['total'] += 1
dates.append(date)
# filter slaves
slave_list = [slave for slave in data if data[slave]['total'] > jobs]
# calculate failure rate only for slaves that we're going to display
for slave in slave_list:
results = data[slave]
fail_rates = calculate_fail_rate(results['success'],
results['retry'],
results['total'])
data[slave]['sfr'] = fail_rates
platforms = {}
# group slaves by platform and calculate platform failure rate
slaves = sorted(data.keys())
for platform, slave_group in groupby(slaves, lambda x: x.rsplit('-', 1)[0]):
slaves = list(slave_group)
# don't calculate failure rate for platform we're not going to show
if not any(slave in slaves for slave in slave_list):
continue
platforms[platform] = {}
results = {}
for label in ['success', 'retry', 'total']:
r = reduce(lambda x, y: x + y,
[data[slave][label] for slave in slaves])
results[label] = r
fail_rates = calculate_fail_rate(results['success'],
results['retry'],
results['total'])
platforms[platform].update(fail_rates)
# remove data that we don't need
for slave in data.keys():
if slave not in slave_list:
del data[slave]
return {'slaves': data,
'platforms': platforms,
'dates': get_date_range(dates),
'disclaimer': info}
def run_seta_details_query():
buildbot = sanitize_bool(request.args.get("buildbot", 0))
branch = sanitize_string(request.args.get("branch", 'mozilla-inbound'))
taskcluster = sanitize_bool(request.args.get("taskcluster", 0))
priority = int(sanitize_string(request.args.get("priority", '5')))
jobnames = JOBSDATA.jobnames_query()
date = str(datetime.now().date())
retVal = {}
retVal[date] = []
jobtype = []
# we only support fx-team, autoland, and mozilla-inbound branch in seta
if (str(branch) in ['fx-team', 'mozilla-inbound', 'autoland']) is not True \
and str(branch) != '':
abort(404)
# For the case of TaskCluster request, we don't care which priority the user request.
# We return jobs depend on the strategy that we return high value jobs as default and
# return all jobs for every 5 push or 90 minutes for that branch.
if request.headers.get('User-Agent', '') == 'TaskCluster':
# we make taskcluster to 1 if it's a request from taskcluster, it's more reasonable and
# can simplify the request url.
taskcluster = 1
# we will return all jobs for every 90 minutes, so the return_all_jobs flag will been
# set to true if the time limit been reached.
return_all_jobs = False
# We should return full job list as a fallback, if it's a request from
# taskcluster and without head_rev or pushlog_id in there
try:
branch_info = session.query(
TaskRequests.counter, TaskRequests.datetime,
TaskRequests.reset_delta).filter(
TaskRequests.branch == branch).all()
except:
branch_info = []
time_of_now = datetime.now()
# If we got nothing related with that branch, we should create it.
if len(branch_info) == 0:
# time_of_lastreset is not a good name anyway :(
# And we treat all branches' reset_delta is 90 seconds, we should find a
# better delta for them in the further.
branch_data = TaskRequests(str(branch), 1, time_of_now,
RESET_DELTA)
try:
session.add(branch_data)
session.commit()
except Exception as error:
LOG.debug(error)
session.rollback()
finally:
session.close()
counter = 1
time_string = time_of_now
reset_delta = RESET_DELTA
# We should update it if that branch had already been stored.
else:
counter, time_string, reset_delta = branch_info[0]
counter += 1
conn = engine.connect()
statement = update(TaskRequests).where(
TaskRequests.branch == branch).values(counter=counter)
conn.execute(statement)
delta = (time_of_now - time_string).total_seconds()
# we should update the time recorder if the elapse time had
# reach the time limit of that branch.
if delta >= reset_delta:
conn = engine.connect()
statement = update(TaskRequests).where(
TaskRequests.branch == branch).values(datetime=time_of_now)
conn.execute(statement)
# we need to set the return_all_jobs flag to true.
return_all_jobs = True
# we query all jobs rather than jobs filter by the requested priority in here,
# Because we need to set the job returning strategy depend on different job priority.
query = session.query(JobPriorities.platform, JobPriorities.buildtype,
JobPriorities.testtype, JobPriorities.priority,
JobPriorities.timeout).all()
for d in query:
# we only return that job if it hasn't reach the timeout limit. And the
# timeout is zero means this job need always running.
if delta < d[4] or d[4] == 0:
# Due to the priority of all high value jobs is 1, and we
# need to return all jobs for every 5 pushes(for now).
if counter % d[3] != 0:
jobtype.append([d[0], d[1], d[2]])
# we need to return all jobs for every 90 minutes, so all jobs will been returned
# if the delta is larger than 5400
elif return_all_jobs:
jobtype.append([d[0], d[1], d[2]])
# We don't care about the timeout variable of job if it's not a taskcluster request.
else:
query = session.query(
JobPriorities.platform,
JobPriorities.buildtype,
JobPriorities.testtype,
JobPriorities.priority,
).all()
# priority = 0; run all the jobs
if priority != 1 and priority != 5:
priority = 0
# Because we store high value jobs in seta table as default,
# so we return low value jobs, means no failure related with this job as default
if priority == 0:
jobtype = JOBSDATA.jobtype_query(
) # All jobs regardless of priority
# priority =5 run all low value jobs
else:
joblist = [job for job in query if job[3] == priority]
for j in joblist:
jobtype.append([j[0], j[1], j[2]])
# TODO: filter out based on buildsystem from database, either 'buildbot' or '*'
if buildbot:
active_jobs = []
# pick up buildbot jobs from job list to faster the filter process
buildbot_jobs = [
job for job in jobnames if job['buildplatform'] == 'buildbot'
]
# find out the correspond job detail information
for job in jobtype:
for j in buildbot_jobs:
if j['name'] == job[2] and j['platform'] == job[0] and j[
'buildtype'] == job[1]:
active_jobs.append(
j['ref_data_name'] if branch is 'mozilla-inbound'
else j['ref_data_name'].
replace('mozilla-inbound', branch))
jobtype = active_jobs
# TODO: filter out based on buildsystem from database, either 'taskcluster' or '*'
if taskcluster:
active_jobs = []
taskcluster_jobs = [
job for job in jobnames if job['buildplatform'] == 'taskcluster'
]
for job in jobtype:
# we need to retranslate the jobtype back to the proper data form after all.
job[2] = job[2].replace('e10s-browser-chrome',
'browser-chrome-e10s')
job[2] = job[2].replace('e10s-devtools-chrome',
'devtools-chrome-e10s')
job[2] = job[2].replace('gl-', 'webgl-')
for j in taskcluster_jobs:
if job[2] in j['name'] and j['platform'] == job[0] and j[
'buildtype'] == job[1]:
active_jobs.append(j['ref_data_name'])
jobtype = active_jobs
retVal[date] = jobtype
return {"jobtypes": retVal}
def add_jobs_to_jobpriority(new_data=None,
priority=1,
timeout=0,
set_expired=False):
added_jobs = []
if not new_data:
return
# TODO: as a perf improvement we can reduce jobs prior to this expensive for loop
for job in new_data['results']:
# TODO: potentially ensure no duplicates in new_data and query once outside the loop
db_data = []
db_data = session.query(JobPriorities.id, JobPriorities.testtype,
JobPriorities.buildtype,
JobPriorities.platform, JobPriorities.priority,
JobPriorities.timeout, JobPriorities.expires,
JobPriorities.buildsystem).all()
platform = parse_platform(job['build_platform'])
if platform == None or platform == "":
continue
testtype = parse_testtype(job['build_system_type'],
job['ref_data_name'], job['platform_option'],
job['job_type_name'])
if testtype == None or testtype == "":
continue
_buildsystem = job["build_system_type"]
found = False
found_id = None
for row in db_data:
if (row[1] == testtype and row[3] == platform
and row[2] == job["platform_option"]):
#TODO: what if we have a race condition with two identical jobs
# verify the build system type is the same, or make it *
found = True
if row[7] != "*" and _buildsystem != row[7]:
_buildsystem = "*"
found_id = row[0]
# We have new jobs from runnablejobs to add to our master list
if not found:
_expired = None
if set_expired:
# set _expired = today + 14 days
# TODO: write test for it
_expired = "%s" % (datetime.datetime.now() +
datetime.timedelta(days=14))
try:
jobpriority = JobPriorities(str(testtype),
str(job["platform_option"]),
str(job["build_platform"]),
priority, timeout, _expired,
_buildsystem)
session.add(jobpriority)
session.commit()
added_jobs.append(job)
except Exception as error:
session.rollback()
logging.warning(error)
finally:
session.close()
elif _buildsystem != job['build_system_type']:
# update table with new buildsystem
conn = engine.connect()
statement = update(JobPriorities)\
.where(JobPriorities.id == found_id)\
.values(buildsystem=_buildsystem)
conn.execute(statement)
return added_jobs
def delete_customer(email_id):
db_session.query(Customer).filter(
Customer.email_id == email_id).delete()
db_session.commit()
db_session.close()
print("Customer deleted successfully. :)")
def get(self):
writers = session.query(Writers.id, Writers.name, Writers.email).all()
date = [{'id': i[0], 'name': i[1], 'email': i[2]} for i in writers]
return date
