def cleanup_tasks():
"""
Find any tasks which haven't checked in within a reasonable time period and
requeue them if necessary.
Additionally remove any old Task entries which are completed.
"""
now = datetime.utcnow()
pending_tasks = Task.query.filter(
Task.status != Status.finished,
Task.date_modified < now - CHECK_TIME,
)
for task in pending_tasks:
task_func = TrackedTask(queue.get_task(task.task_name))
task_func.delay(
task_id=task.task_id.hex,
parent_task_id=task.parent_id.hex if task.parent_id else None,
**task.data['kwargs'])
deleted = Task.query.filter(
Task.status == Status.finished,
Task.date_modified < now - EXPIRE_TIME,
# Filtering by date_created isn't necessary, but it allows us to filter using an index on
# a value that doesn't update, which makes our deletion more efficient.
Task.date_created < now - EXPIRE_TIME,
).delete(synchronize_session=False)
statsreporter.stats().incr('tasks_deleted', deleted)
def cleanup_tasks():
"""
Find any tasks which haven't checked in within a reasonable time period and
requeue them if necessary.
Additionally remove any old Task entries which are completed.
"""
now = datetime.utcnow()
pending_tasks = Task.query.filter(
Task.status != Status.finished,
Task.date_modified < now - CHECK_TIME,
)
for task in pending_tasks:
task_func = TrackedTask(queue.get_task(task.task_name))
task_func.delay(
task_id=task.task_id.hex,
parent_task_id=task.parent_id.hex if task.parent_id else None,
**task.data['kwargs']
)
deleted = Task.query.filter(
Task.status == Status.finished,
Task.date_modified < now - EXPIRE_TIME,
# Filtering by date_created isn't necessary, but it allows us to filter using an index on
# a value that doesn't update, which makes our deletion more efficient.
Task.date_created < now - EXPIRE_TIME,
).delete(synchronize_session=False)
statsreporter.stats().incr('tasks_deleted', deleted)
def _start(self, tag, attrs):
if tag == 'unittest-results':
raise ArtifactParseError('Bitten is not supported.')
else:
self._set_subparser(XunitParser(self.step, self._parser))
statsreporter.stats().incr('new_xunit_result_file')
self._parser.StartElementHandler(tag, attrs)
def get(self, path=''):
statsreporter.stats().incr('homepage_view')
if current_app.config['SENTRY_DSN'] and False:
parsed = urlparse.urlparse(current_app.config['SENTRY_DSN'])
dsn = '%s://%s@%s/%s' % (
parsed.scheme.rsplit('+', 1)[-1],
parsed.username,
parsed.hostname + (':%s' % (parsed.port,) if parsed.port else ''),
parsed.path,
)
else:
dsn = None
dev_js_should_hit_host = current_app.config['DEV_JS_SHOULD_HIT_HOST']
# use new react code
if path.startswith("experimental"):
return render_template('experimental.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'VERSION': changes.get_version(),
'DEV_JS_SHOULD_HIT_HOST': dev_js_should_hit_host
})
return render_template('index.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'VERSION': changes.get_version(),
'DEV_JS_SHOULD_HIT_HOST': dev_js_should_hit_host
})
def _incr(name):
"""Helper to increments a stats counter.
Mostly exists to ease mocking in tests.
Args:
name (str): Name of counter to increment.
"""
statsreporter.stats().incr(name)
def log_page_perf(self, perf_stats):
page_load = 'full' if perf_stats['fullPageLoad'] else 'ajax'
key = "changes_page_perf_load_{}_name_{}".format(
page_load, self.make_fuzzy_url(perf_stats['url']))
statsreporter.stats().log_timing(
self.strip_illegal_chars(key),
perf_stats['endTime'] - perf_stats['startTime'])
def create_build(
project,
collection_id,
label,
target,
message,
author,
change=None,
patch=None,
cause=None,
source=None,
sha=None,
source_data=None,
tag=None,
snapshot_id=None,
no_snapshot=False,
):
assert sha or source
repository = project.repository
if source is None:
if patch:
source, _ = get_or_create(
Source,
where={"patch": patch},
defaults={"repository": repository, "revision_sha": sha, "data": source_data or {}},
)
else:
source, _ = get_or_create(
Source,
where={"repository": repository, "patch": None, "revision_sha": sha},
defaults={"data": source_data or {}},
)
statsreporter.stats().incr("new_api_build")
build = Build(
project=project,
project_id=project.id,
collection_id=collection_id,
source=source,
source_id=source.id if source else None,
status=Status.queued,
author=author,
author_id=author.id if author else None,
label=label,
target=target,
message=message,
cause=cause,
tags=[tag] if tag else [],
)
db.session.add(build)
db.session.commit()
execute_build(build=build, snapshot_id=snapshot_id, no_snapshot=no_snapshot)
return build
def respond(self, context, status_code=200, serialize=True, serializers=None,
links=None):
if serialize:
data = self.serialize(context, serializers)
else:
data = context
response = Response(
_as_json(data),
mimetype='application/json',
status=status_code,
)
if links:
response.headers['Link'] = ', '.join(links)
response.headers['changes-api-class'] = self.__class__.__name__
# do some performance logging / send perf data back to the client
timer_name = "changes_api_server_perf_method_{}_class_{}".format(
request.method, self.__class__.__name__)
time_taken = time() - self.start_time
statsreporter.stats().log_timing(timer_name, time_taken * 1000)
response.headers['changes-server-time'] = time_taken
# how much time did we spend waiting on the db
db_time_in_sec = sum([q.duration for q in get_debug_queries()])
db_timer_name = "changes_api_total_db_time_method_{}_class_{}".format(
request.method, self.__class__.__name__)
statsreporter.stats().log_timing(db_timer_name, db_time_in_sec * 1000)
response.headers['changes-server-db-time'] = db_time_in_sec
return response
def verify_all_children(self):
task_list = list(Task.query.filter(
Task.parent_id == self.task_id,
Task.status != Status.finished,
))
if not task_list:
return Status.finished
current_datetime = datetime.utcnow()
need_expire = set()
need_run = set()
has_pending = False
for task in task_list:
if self.needs_expired(task):
need_expire.add(task)
continue
has_pending = True
if self.needs_requeued(task) and 'kwargs' in task.data:
need_run.add(task)
if need_expire:
Task.query.filter(
Task.id.in_([n.id for n in need_expire]),
).update({
Task.date_modified: current_datetime,
Task.date_finished: current_datetime,
Task.status: Status.finished,
Task.result: Result.aborted,
}, synchronize_session=False)
db.session.commit()
if need_run:
for task in need_run:
child_kwargs = task.data['kwargs'].copy()
child_kwargs['parent_task_id'] = task.parent_id.hex
child_kwargs['task_id'] = task.task_id.hex
queue.delay(task.task_name, kwargs=child_kwargs)
Task.query.filter(
Task.id.in_([n.id for n in need_run]),
).update({
Task.date_modified: current_datetime,
}, synchronize_session=False)
db.session.commit()
for name, count in Counter((task.task_name for task in need_run)).iteritems():
statsreporter.stats().incr('task_revived_by_parent_' + name, count)
if has_pending:
status = Status.in_progress
else:
status = Status.finished
return status
def _sync_step_from_queue(self, step):
# TODO(dcramer): when we hit a NotFound in the queue, maybe we should
# attempt to scrape the list of jobs for a matching CHANGES_BID, as this
# doesn't explicitly mean that the job doesn't exist.
try:
item = self._get_json_response(
step.data['master'],
'/queue/item/{}'.format(step.data['item_id']),
)
except NotFound:
step.status = Status.finished
step.result = Result.infra_failed
db.session.add(step)
self.logger.exception("Queued step not found in queue: {} (build: {})".format(step.id, step.job.build_id))
statsreporter.stats().incr('jenkins_item_not_found_in_queue')
return
if item.get('executable'):
build_no = item['executable']['number']
step.data['queued'] = False
step.data['build_no'] = build_no
step.data['uri'] = item['executable']['url']
db.session.add(step)
if item['blocked']:
step.status = Status.queued
db.session.add(step)
elif item.get('cancelled') and not step.data.get('build_no'):
step.status = Status.finished
step.result = Result.aborted
db.session.add(step)
elif item.get('executable'):
return self._sync_step_from_active(step)
def _incr(name):
"""Helper to increments a stats counter.
Mostly exists to ease mocking in tests.
Args:
name (str): Name of counter to increment.
"""
statsreporter.stats().incr(name)
def get(self, path=''):
# get user options, e.g. colorblind
current_user = get_current_user()
user_options = {}
if current_user:
user_options = dict(
db.session.query(
ItemOption.name,
ItemOption.value,
).filter(ItemOption.item_id == current_user.id, ))
statsreporter.stats().incr('homepage_view')
if current_app.config['SENTRY_DSN'] and False:
parsed = urlparse.urlparse(current_app.config['SENTRY_DSN'])
dsn = '%s://%s@%s/%s' % (
parsed.scheme.rsplit('+', 1)[-1],
parsed.username,
parsed.hostname + (':%s' %
(parsed.port, ) if parsed.port else ''),
parsed.path,
)
else:
dsn = None
# variables to ship down to the webapp
use_another_host = current_app.config['WEBAPP_USE_ANOTHER_HOST']
# if we have custom js, embed it in the html (making sure we
# only do one file read in prod).
fetch_custom_js = (current_app.config['WEBAPP_CUSTOM_JS']
and (current_app.debug or not IndexView.custom_js))
if fetch_custom_js:
IndexView.custom_js = open(
current_app.config['WEBAPP_CUSTOM_JS']).read()
disable_custom = request.args and "disable_custom" in request.args
return render_template(
'webapp.html', **{
'SENTRY_PUBLIC_DSN':
dsn,
'RELEASE_INFO':
changes.get_revision_info(),
'WEBAPP_USE_ANOTHER_HOST':
use_another_host,
'WEBAPP_CUSTOM_JS':
(IndexView.custom_js if not disable_custom else None),
'USE_PACKAGED_JS':
not current_app.debug,
'HAS_CUSTOM_CSS': (current_app.config['WEBAPP_CUSTOM_CSS']
and not disable_custom),
'IS_DEBUG':
current_app.debug,
'PHABRICATOR_LINK_HOST':
current_app.config['PHABRICATOR_LINK_HOST'],
'COLORBLIND': (user_options.get('user.colorblind')
and user_options.get('user.colorblind') != '0'),
})
def log_page_perf(self, perf_stats):
page_load = 'full' if perf_stats['fullPageLoad'] else 'ajax'
key = "changes_page_perf_load_{}_name_{}".format(
page_load,
self.make_fuzzy_url(perf_stats['url']))
statsreporter.stats().log_timing(
key,
perf_stats['endTime'] - perf_stats['startTime'])
def update_step(self, step):
# type: (JobStep) -> None
if step.status == Status.allocated and step.last_heartbeat:
duration = utcnow() - step.last_heartbeat
if duration.total_seconds() >= current_app.config['JOBSTEP_ALLOCATION_TIMEOUT_SECONDS']:
# Allocation has timed out; move back to being elligible for allocation.
step.status = Status.pending_allocation
statsreporter.stats().incr('jobstep_allocation_timeout')
def _start(self, tag, attrs):
if tag == 'unittest-results':
self._set_subparser(BittenParser(self.step, self._parser))
statsreporter.stats().incr('new_bitten_result_file')
else:
self._set_subparser(XunitParser(self.step, self._parser))
statsreporter.stats().incr('new_xunit_result_file')
self._parser.StartElementHandler(tag, attrs)
def _report_snapshot_downgrade(project):
"""Reports that we've downgraded a snapshot.
Mostly abstracted out to ease testing.
"""
statsreporter.stats().incr("downgrade")
# Warning is arguable, since a downgrade isn't a problem, just needing one
# likely is. This is just the easiest way to surface this event at the moment.
logging.warning('Snapshot downgrade for project %s', project.slug)
def _report_snapshot_downgrade(project):
"""Reports that we've downgraded a snapshot.
Mostly abstracted out to ease testing.
"""
statsreporter.stats().incr("downgrade")
# Warning is arguable, since a downgrade isn't a problem, just needing one
# likely is. This is just the easiest way to surface this event at the moment.
logging.warning('Snapshot downgrade for project %s', project.slug)
def get(self, path=''):
# get user options, e.g. colorblind
current_user = get_current_user()
user_options = {}
if current_user:
user_options = dict(db.session.query(
ItemOption.name, ItemOption.value,
).filter(
ItemOption.item_id == current_user.id,
))
statsreporter.stats().incr('homepage_view')
if current_app.config['SENTRY_DSN'] and False:
parsed = urlparse.urlparse(current_app.config['SENTRY_DSN'])
dsn = '%s://%s@%s/%s' % (
parsed.scheme.rsplit('+', 1)[-1],
parsed.username,
parsed.hostname + (':%s' % (parsed.port,) if parsed.port else ''),
parsed.path,
)
else:
dsn = None
# variables to ship down to the webapp
use_another_host = current_app.config['WEBAPP_USE_ANOTHER_HOST']
# if we have custom js, embed it in the html (making sure we
# only do one file read in prod).
fetch_custom_js = (current_app.config['WEBAPP_CUSTOM_JS'] and
(current_app.debug or not IndexView.custom_js))
if fetch_custom_js:
IndexView.custom_js = open(current_app.config['WEBAPP_CUSTOM_JS']).read()
disable_custom = request.args and "disable_custom" in request.args
# use new react code
if self.use_v2:
return render_template('webapp.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'RELEASE_INFO': changes.get_revision_info(),
'WEBAPP_USE_ANOTHER_HOST': use_another_host,
'WEBAPP_CUSTOM_JS': (IndexView.custom_js if
not disable_custom else None),
'USE_PACKAGED_JS': not current_app.debug,
'HAS_CUSTOM_CSS': (current_app.config['WEBAPP_CUSTOM_CSS'] and
not disable_custom),
'IS_DEBUG': current_app.debug,
'PHABRICATOR_HOST': current_app.config['PHABRICATOR_HOST'],
'COLORBLIND': (user_options.get('user.colorblind') and
user_options.get('user.colorblind') != '0'),
})
return render_template('index.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'VERSION': changes.get_version(),
'WEBAPP_USE_ANOTHER_HOST': use_another_host
})
def update_step(self, step):
# type: (JobStep) -> None
if step.status == Status.allocated and step.last_heartbeat:
duration = utcnow() - step.last_heartbeat
if duration.total_seconds(
) >= current_app.config['JOBSTEP_ALLOCATION_TIMEOUT_SECONDS']:
# Allocation has timed out; move back to being elligible for allocation.
step.status = Status.pending_allocation
statsreporter.stats().incr('jobstep_allocation_timeout')
def log_api_perf(self, perf_stats):
api_key = "changes_api_client_perf_method_{}_class_{}"
for _, api_data in perf_stats['apiCalls'].iteritems():
# this can happen when we get a 404 from an api endpoint
if 'endTime' not in api_data:
continue
duration = api_data['endTime'] - api_data['startTime']
statsreporter.stats().log_timing(
api_key.format(api_data['apiMethod'], api_data['apiName']),
duration)
def get(self, path=""):
# get user options, e.g. colorblind
current_user = get_current_user()
user_options = {}
if current_user:
user_options = dict(
db.session.query(ItemOption.name, ItemOption.value).filter(ItemOption.item_id == current_user.id)
)
statsreporter.stats().incr("homepage_view")
if current_app.config["SENTRY_DSN"] and False:
parsed = urlparse.urlparse(current_app.config["SENTRY_DSN"])
dsn = "%s://%s@%s/%s" % (
parsed.scheme.rsplit("+", 1)[-1],
parsed.username,
parsed.hostname + (":%s" % (parsed.port,) if parsed.port else ""),
parsed.path,
)
else:
dsn = None
# variables to ship down to the webapp
use_another_host = current_app.config["WEBAPP_USE_ANOTHER_HOST"]
# if we have custom js, embed it in the html (making sure we
# only do one file read in prod).
fetch_custom_js = current_app.config["WEBAPP_CUSTOM_JS"] and (current_app.debug or not IndexView.custom_js)
if fetch_custom_js:
IndexView.custom_js = open(current_app.config["WEBAPP_CUSTOM_JS"]).read()
disable_custom = request.args and "disable_custom" in request.args
# use new react code
if self.use_v2:
return render_template(
"webapp.html",
**{
"SENTRY_PUBLIC_DSN": dsn,
"RELEASE_INFO": changes.get_revision_info(),
"WEBAPP_USE_ANOTHER_HOST": use_another_host,
"WEBAPP_CUSTOM_JS": (IndexView.custom_js if not disable_custom else None),
"USE_PACKAGED_JS": not current_app.debug,
"HAS_CUSTOM_CSS": (current_app.config["WEBAPP_CUSTOM_CSS"] and not disable_custom),
"IS_DEBUG": current_app.debug,
"PHABRICATOR_LINK_HOST": current_app.config["PHABRICATOR_LINK_HOST"],
"COLORBLIND": (user_options.get("user.colorblind") and user_options.get("user.colorblind") != "0"),
}
)
return render_template(
"index.html",
**{"SENTRY_PUBLIC_DSN": dsn, "VERSION": changes.get_version(), "WEBAPP_USE_ANOTHER_HOST": use_another_host}
)
def log_api_perf(self, perf_stats):
api_key = "changes_api_client_perf_method_{}_class_{}"
for _, api_data in perf_stats['apiCalls'].iteritems():
# this can happen when we get a 404 from an api endpoint
if 'endTime' not in api_data:
continue
duration = api_data['endTime'] - api_data['startTime']
statsreporter.stats().log_timing(
self.strip_illegal_chars(
api_key.format(api_data['apiMethod'],
api_data['apiName'])), duration)
def create_build(project, collection_id, label, target, message, author,
change=None, patch=None, cause=None, source=None, sha=None,
source_data=None, tag=None, snapshot_id=None, no_snapshot=False,
selective_testing_policy=None):
assert sha or source
repository = project.repository
if source is None:
if patch:
source, _ = get_or_create(Source, where={
'patch': patch,
}, defaults={
'repository': repository,
'revision_sha': sha,
'data': source_data or {},
})
else:
source, _ = get_or_create(Source, where={
'repository': repository,
'patch': None,
'revision_sha': sha,
}, defaults={
'data': source_data or {},
})
statsreporter.stats().incr('new_api_build')
build = Build(
project=project,
project_id=project.id,
collection_id=collection_id,
source=source,
source_id=source.id if source else None,
status=Status.queued,
author=author,
author_id=author.id if author else None,
label=label,
target=target,
message=message,
cause=cause,
tags=[tag] if tag else [],
selective_testing_policy=selective_testing_policy,
)
db.session.add(build)
db.session.commit()
execute_build(build=build, snapshot_id=snapshot_id, no_snapshot=no_snapshot)
return build
def log_timing(self, command, start_time):
repo_type = 'unknown'
classname = self.__class__.__name__
if "Git" in classname:
repo_type = 'git'
elif "Mercurial" in classname:
repo_type = 'hg'
timer_name = "changes_vcs_perf_{}_command_{}".format(
repo_type, command)
time_taken = time() - start_time
statsreporter.stats().log_timing(timer_name, time_taken * 1000)
def log_timing(self, command, start_time):
repo_type = 'unknown'
classname = self.__class__.__name__
if "Git" in classname:
repo_type = 'git'
elif "Mercurial" in classname:
repo_type = 'hg'
timer_name = "changes_vcs_perf_{}_command_{}".format(
repo_type, command)
time_taken = time() - start_time
statsreporter.stats().log_timing(timer_name, time_taken * 1000)
def _find_and_retry_jobsteps(phase, implementation):
# phase.steps is ordered by date_started, so we retry the oldest jobsteps first
should_retry = [s for s in phase.steps if _should_retry_jobstep(s)]
if not should_retry:
return
already_retried = JobStep.query.filter(JobStep.job == phase.job,
JobStep.replacement_id.isnot(None)).count()
max_retry = current_app.config['JOBSTEP_RETRY_MAX'] - already_retried
for step in should_retry:
if max_retry <= 0:
break
newstep = implementation.create_replacement_jobstep(step)
if newstep:
statsreporter.stats().incr('jobstep_replaced')
max_retry -= 1
def _find_and_retry_jobsteps(phase, implementation):
# phase.steps is ordered by date_started, so we retry the oldest jobsteps first
should_retry = [s for s in phase.steps if _should_retry_jobstep(s)]
if not should_retry:
return
already_retried = JobStep.query.filter(
JobStep.job == phase.job, JobStep.replacement_id.isnot(None)).count()
max_retry = current_app.config['JOBSTEP_RETRY_MAX'] - already_retried
for step in should_retry:
if max_retry <= 0:
break
newstep = implementation.create_replacement_jobstep(step)
if newstep:
statsreporter.stats().incr('jobstep_replaced')
max_retry -= 1
def _sync_step_from_queue(self, step):
try:
item = self._get_json_response(
step.data['master'],
'/queue/item/{}'.format(step.data['item_id']),
)
except NotFound:
# The build might've left the Jenkins queue since we last checked; look for the build_no of the
# running build.
build_no = self._find_build_no(step.data['master'],
step.data['job_name'],
changes_bid=step.id.hex)
if build_no:
step.data['queued'] = False
step.data['build_no'] = build_no
db.session.add(step)
self._sync_step_from_active(step)
return
step.status = Status.finished
step.result = Result.infra_failed
db.session.add(step)
self.logger.exception(
"Queued step not found in queue: {} (build: {})".format(
step.id, step.job.build_id))
statsreporter.stats().incr('jenkins_item_not_found_in_queue')
return
if item.get('executable'):
build_no = item['executable']['number']
step.data['queued'] = False
step.data['build_no'] = build_no
step.data['uri'] = item['executable']['url']
db.session.add(step)
if item['blocked']:
step.status = Status.queued
db.session.add(step)
elif item.get('cancelled') and not step.data.get('build_no'):
step.status = Status.finished
step.result = Result.aborted
db.session.add(step)
elif item.get('executable'):
self._sync_step_from_active(step)
return
def sync_grouper():
# type: () -> None
"""This function is meant as a Celery task. It connects to Grouper, and
does two sets of syncs:
- global admin
- project-level admins
"""
try:
admin_emails = _get_admin_emails_from_grouper()
_sync_admin_users(admin_emails)
project_admin_mapping = _get_project_admin_mapping_from_grouper()
_sync_project_admin_users(project_admin_mapping)
except Exception:
logger.exception("An error occurred during Grouper sync.")
statsreporter.stats().set_gauge('grouper_sync_error', 1)
raise
else:
statsreporter.stats().set_gauge('grouper_sync_error', 0)
def _report_jobstep_result(step):
"""To be called once we're done syncing a JobStep to report the result for monitoring.
Args:
step (JobStep): The JobStep to report the result of.
"""
labels = {
Result.unknown: 'unknown',
Result.passed: 'passed',
Result.failed: 'failed',
Result.infra_failed: 'infra_failed',
Result.aborted: 'aborted',
Result.skipped: 'skipped',
}
label = labels.get(step.result, 'OTHER')
# TODO(kylec): Include the project slug in the metric so we can
# track on a per-project basis if needed.
statsreporter.stats().incr('jobstep_result_' + label)
def _report_jobstep_result(step):
"""To be called once we're done syncing a JobStep to report the result for monitoring.
Args:
step (JobStep): The JobStep to report the result of.
"""
labels = {
Result.unknown: 'unknown',
Result.passed: 'passed',
Result.failed: 'failed',
Result.infra_failed: 'infra_failed',
Result.aborted: 'aborted',
Result.skipped: 'skipped',
}
label = labels.get(step.result, 'OTHER')
# TODO(kylec): Include the project slug in the metric so we can
# track on a per-project basis if needed.
statsreporter.stats().incr('jobstep_result_' + label)
def _report_lag(self, first_run_time):
# type: (datetime) -> None
"""
Reports the time it took from creation to just before the first run of the Task; on subsequent
runs no reporting will occur.
Must be called before the `Task.date_modified` is updated.
Args:
first_run_time (datetime): When the task started running.
"""
t = Task.query.filter(Task.task_name == self.task_name,
Task.task_id == self.task_id,
Task.parent_id == self.parent_id).first()
# Ensure the task exists, and that the creation and modification date
# are the same (meaning we're at the first run).
if t and t.date_created == t.date_modified:
lag_ms = (first_run_time - t.date_created).total_seconds() * 1000
statsreporter.stats().log_timing(
'first_execution_lag_' + self.task_name, lag_ms)
def get(self, path=''):
statsreporter.stats().incr('homepage_view')
if current_app.config['SENTRY_DSN'] and False:
parsed = urlparse.urlparse(current_app.config['SENTRY_DSN'])
dsn = '%s://%s@%s/%s' % (
parsed.scheme.rsplit('+', 1)[-1],
parsed.username,
parsed.hostname + (':%s' %
(parsed.port, ) if parsed.port else ''),
parsed.path,
)
else:
dsn = None
return render_template(
'index.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'VERSION': changes.get_version(),
})
def get(self, path=""):
statsreporter.stats().incr("homepage_view")
if current_app.config["SENTRY_DSN"] and False:
parsed = urlparse.urlparse(current_app.config["SENTRY_DSN"])
dsn = "%s://%s@%s/%s" % (
parsed.scheme.rsplit("+", 1)[-1],
parsed.username,
parsed.hostname + (":%s" % (parsed.port,) if parsed.port else ""),
parsed.path,
)
else:
dsn = None
# variables to ship down to the webapp
webapp_use_another_host = current_app.config["WEBAPP_USE_ANOTHER_HOST"]
# note that we're only shipping down the filename!
webapp_customized_content = None
if current_app.config["WEBAPP_CUSTOMIZED_CONTENT_FILE"]:
webapp_customized_content = open(current_app.config["WEBAPP_CUSTOMIZED_CONTENT_FILE"]).read()
# use new react code
if self.use_v2:
return render_template(
"webapp.html",
**{
"SENTRY_PUBLIC_DSN": dsn,
"RELEASE_INFO": changes.get_revision_info(),
"WEBAPP_USE_ANOTHER_HOST": webapp_use_another_host,
"WEBAPP_CUSTOMIZED_CONTENT": webapp_customized_content,
"USE_PACKAGED_JS": not current_app.debug,
"IS_DEBUG": current_app.debug,
}
)
return render_template(
"index.html",
**{
"SENTRY_PUBLIC_DSN": dsn,
"VERSION": changes.get_version(),
"WEBAPP_USE_ANOTHER_HOST": webapp_use_another_host,
}
)
def _report_lag(self, first_run_time):
# type: (datetime) -> None
"""
Reports the time it took from creation to just before the first run of the Task; on subsequent
runs no reporting will occur.
Must be called before the `Task.date_modified` is updated.
Args:
first_run_time (datetime): When the task started running.
"""
t = Task.query.filter(
Task.task_name == self.task_name,
Task.task_id == self.task_id,
Task.parent_id == self.parent_id
).first()
# Ensure the task exists, and that the creation and modification date
# are the same (meaning we're at the first run).
if t and t.date_created == t.date_modified:
lag_ms = (first_run_time - t.date_created).total_seconds() * 1000
statsreporter.stats().log_timing('first_execution_lag_' + self.task_name, lag_ms)
def respond(self, context, status_code=200, serialize=True, serializers=None,
links=None):
if serialize:
data = self.serialize(context, serializers)
else:
data = context
response = Response(
as_json(data),
mimetype='application/json',
status=status_code,
)
if links:
response.headers['Link'] = ', '.join(links)
timer_name = "http-method-{}_api-view-class-{}".format(request.method,
self.__class__.__name__)
statsreporter.stats().log_timing(timer_name, time() - self.start_time)
return response
def _sync_step_from_queue(self, step):
try:
item = self._get_json_response(
step.data['master'],
'/queue/item/{}'.format(step.data['item_id']),
)
except NotFound:
# The build might've left the Jenkins queue since we last checked; look for the build_no of the
# running build.
build_no = self._find_build_no(step.data['master'], step.data['job_name'], changes_bid=step.id.hex)
if build_no:
step.data['queued'] = False
step.data['build_no'] = build_no
db.session.add(step)
self._sync_step_from_active(step)
return
step.status = Status.finished
step.result = Result.infra_failed
db.session.add(step)
self.logger.exception("Queued step not found in queue: {} (build: {})".format(step.id, step.job.build_id))
statsreporter.stats().incr('jenkins_item_not_found_in_queue')
return
if item.get('executable'):
build_no = item['executable']['number']
step.data['queued'] = False
step.data['build_no'] = build_no
step.data['uri'] = item['executable']['url']
db.session.add(step)
if item['blocked']:
step.status = Status.queued
db.session.add(step)
elif item.get('cancelled') and not step.data.get('build_no'):
step.status = Status.finished
step.result = Result.aborted
db.session.add(step)
elif item.get('executable'):
self._sync_step_from_active(step)
return
def get(self, path=''):
statsreporter.stats().incr('homepage_view')
if current_app.config['SENTRY_DSN'] and False:
parsed = urlparse.urlparse(current_app.config['SENTRY_DSN'])
dsn = '%s://%s@%s/%s' % (
parsed.scheme.rsplit('+', 1)[-1],
parsed.username,
parsed.hostname + (':%s' %
(parsed.port, ) if parsed.port else ''),
parsed.path,
)
else:
dsn = None
# variables to ship down to the webapp
webapp_use_another_host = current_app.config['WEBAPP_USE_ANOTHER_HOST']
# note that we're only shipping down the filename!
webapp_customized_content = None
if current_app.config['WEBAPP_CUSTOMIZED_CONTENT_FILE']:
webapp_customized_content = open(
current_app.config['WEBAPP_CUSTOMIZED_CONTENT_FILE']).read()
# use new react code
if self.use_v2:
return render_template(
'webapp.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'RELEASE_INFO': changes.get_revision_info(),
'WEBAPP_USE_ANOTHER_HOST': webapp_use_another_host,
'WEBAPP_CUSTOMIZED_CONTENT': webapp_customized_content,
'USE_PACKAGED_JS': not current_app.debug,
'IS_DEBUG': current_app.debug
})
return render_template(
'index.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'VERSION': changes.get_version(),
'WEBAPP_USE_ANOTHER_HOST': webapp_use_another_host
})
def post(self):
perf_stats = request.get_json(True)
key_prefix = ['client_perf']
if current_app.config['DEBUG']:
key_prefix.append('dev')
key_prefix.append('initial' if perf_stats['initial'] else 'switch')
# record total time per page
page_key_prefix = key_prefix[:]
page_key_prefix.append('page')
page_key = self.url_to_key(perf_stats['url'], page_key_prefix)
page_duration = perf_stats['endTime'] - perf_stats['startTime']
statsreporter.stats().log_timing(page_key, page_duration)
# record stats for each api call
url_key_prefix = key_prefix[:] # don't append api, already there
for url, times in perf_stats['apiCalls'].iteritems():
key = self.url_to_key(url, url_key_prefix)
start_time = times['startTime'] - perf_stats['startTime']
# this can happen when we get a 404 from an api endpoint
if 'endTime' not in times:
continue
duration = times['endTime'] - times['startTime']
statsreporter.stats().log_timing(key, duration)
statsreporter.stats().set_gauge(key + '_start', start_time)
return '', 200
def get(self, path=''):
statsreporter.stats().incr('homepage_view')
if current_app.config['SENTRY_DSN'] and False:
parsed = urlparse.urlparse(current_app.config['SENTRY_DSN'])
dsn = '%s://%s@%s/%s' % (
parsed.scheme.rsplit('+', 1)[-1],
parsed.username,
parsed.hostname + (':%s' % (parsed.port,) if parsed.port else ''),
parsed.path,
)
else:
dsn = None
# variables to ship down to the webapp
use_another_host = current_app.config['WEBAPP_USE_ANOTHER_HOST']
# if we have custom js, embed it in the html (making sure we
# only do one file read in prod).
fetch_custom_js = (current_app.config['WEBAPP_CUSTOM_JS'] and
(current_app.debug or not IndexView.custom_js))
if fetch_custom_js:
IndexView.custom_js = open(current_app.config['WEBAPP_CUSTOM_JS']).read()
# use new react code
if self.use_v2:
return render_template('webapp.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'RELEASE_INFO': changes.get_revision_info(),
'WEBAPP_USE_ANOTHER_HOST': use_another_host,
'WEBAPP_CUSTOM_JS': IndexView.custom_js,
'USE_PACKAGED_JS': not current_app.debug,
'IS_DEBUG': current_app.debug
})
return render_template('index.html', **{
'SENTRY_PUBLIC_DSN': dsn,
'VERSION': changes.get_version(),
'WEBAPP_USE_ANOTHER_HOST': use_another_host
})
def clean_project_tests(project, from_date, chunk_size, num_days=None):
# type: (Project, datetime, timedelta, int) -> int
"""Deletes old tests from a project and returns number of rows deleted.
An old test is a test older than num_days or the project's history.test-retention-days
compared to the `from_date`
chunk_size bounds how far to back to look from num_days ago to have some control over
how long this function runs.
"""
if chunk_size <= timedelta(minutes=0):
logger.warning('The minutes worth of tests to delete is %s but it must be positive.' %
chunk_size)
return 0
test_retention_days = num_days or float(
ProjectOptionsHelper.get_option(project, 'history.test-retention-days') or
DEFAULT_TEST_RETENTION_DAYS
)
if test_retention_days < MINIMUM_TEST_RETENTION_DAYS:
logger.warning(
'Test retention days for project %s is %d, which is less than the minimum of %d. '
'Not cleaning tests for this project.' %
(project.slug, test_retention_days, MINIMUM_TEST_RETENTION_DAYS))
return 0
test_delete_date = from_date - timedelta(days=test_retention_days)
test_delete_date_limit = test_delete_date - chunk_size
rows_deleted = db.session.query(TestCase).filter(
TestCase.project_id == project.id,
TestCase.date_created < test_delete_date,
TestCase.date_created >= test_delete_date_limit,
).delete()
db.session.commit()
statsreporter.stats().incr('count_tests_deleted', rows_deleted)
return rows_deleted
def _sync_step_from_queue(self, step):
try:
item = self._get_json_response(step.data["master"], "/queue/item/{}".format(step.data["item_id"]))
except NotFound:
# The build might've left the Jenkins queue since we last checked; look for the build_no of the
# running build.
build_no = self._find_build_no(step.data["master"], step.data["job_name"], changes_bid=step.id.hex)
if build_no:
step.data["queued"] = False
step.data["build_no"] = build_no
db.session.add(step)
self._sync_step_from_active(step)
return
step.status = Status.finished
step.result = Result.infra_failed
db.session.add(step)
self.logger.exception("Queued step not found in queue: {} (build: {})".format(step.id, step.job.build_id))
statsreporter.stats().incr("jenkins_item_not_found_in_queue")
return
if item.get("executable"):
build_no = item["executable"]["number"]
step.data["queued"] = False
step.data["build_no"] = build_no
step.data["uri"] = item["executable"]["url"]
db.session.add(step)
if item["blocked"]:
step.status = Status.queued
db.session.add(step)
elif item.get("cancelled") and not step.data.get("build_no"):
step.status = Status.finished
step.result = Result.aborted
db.session.add(step)
elif item.get("executable"):
self._sync_step_from_active(step)
return
def clean_project_tests(project, from_date, chunk_size, num_days=None):
# type: (Project, datetime, timedelta, int) -> int
"""Deletes old tests from a project and returns number of rows deleted.
An old test is a test older than num_days or the project's history.test-retention-days
compared to the `from_date`
chunk_size bounds how far to back to look from num_days ago to have some control over
how long this function runs.
"""
if chunk_size <= timedelta(minutes=0):
logger.warning(
'The minutes worth of tests to delete is %s but it must be positive.'
% chunk_size)
return 0
test_retention_days = num_days or float(
ProjectOptionsHelper.get_option(project, 'history.test-retention-days')
or DEFAULT_TEST_RETENTION_DAYS)
if test_retention_days < MINIMUM_TEST_RETENTION_DAYS:
logger.warning(
'Test retention days for project %s is %d, which is less than the minimum of %d. '
'Not cleaning tests for this project.' %
(project.slug, test_retention_days, MINIMUM_TEST_RETENTION_DAYS))
return 0
test_delete_date = from_date - timedelta(days=test_retention_days)
test_delete_date_limit = test_delete_date - chunk_size
rows_deleted = db.session.query(TestCase).filter(
TestCase.project_id == project.id,
TestCase.date_created < test_delete_date,
TestCase.date_created >= test_delete_date_limit,
).delete()
db.session.commit()
statsreporter.stats().incr('count_tests_deleted', rows_deleted)
return rows_deleted
def respond(self, context, status_code=200, serialize=True, serializers=None,
links=None):
if serialize:
data = self.serialize(context, serializers)
else:
data = context
response = Response(
as_json(data),
mimetype='application/json',
status=status_code,
)
if links:
response.headers['Link'] = ', '.join(links)
response.headers['changes-api-class'] = self.__class__.__name__
timer_name = "changes_api_server_perf_method_{}_class_{}".format(
request.method, self.__class__.__name__)
time_taken = time() - self.start_time
statsreporter.stats().log_timing(timer_name, time_taken * 1000)
response.headers['changes-server-time'] = time_taken
return response
def clean_project_tests(project, current_date, num_days=None):
test_retention_days = num_days or float(
ProjectOptionsHelper.get_option(project, 'history.test-retention-days') or
DEFAULT_TEST_RETENTION_DAYS
)
if test_retention_days < MINIMUM_TEST_RETENTION_DAYS:
logger.warning(
'Test retention days for project %s is %d, which is less than the minimum of %d. '
'Not cleaning tests for this project.' %
(project.slug, test_retention_days, MINIMUM_TEST_RETENTION_DAYS))
return 0
test_delete_date = current_date - timedelta(days=test_retention_days)
rows_deleted = db.session.query(TestCase).filter(
TestCase.project_id == project.id,
TestCase.date_created < test_delete_date,
).delete()
db.session.commit()
statsreporter.stats().incr('count_tests_deleted', rows_deleted)
return rows_deleted
def respond(self,
context,
status_code=200,
serialize=True,
serializers=None,
links=None):
if serialize:
data = self.serialize(context, serializers)
else:
data = context
response = Response(
_as_json(data),
mimetype='application/json',
status=status_code,
)
if links:
response.headers['Link'] = ', '.join(links)
response.headers['changes-api-class'] = self.__class__.__name__
# do some performance logging / send perf data back to the client
timer_name = "changes_api_server_perf_method_{}_class_{}".format(
request.method, self.__class__.__name__)
time_taken = time() - self.start_time
statsreporter.stats().log_timing(timer_name, time_taken * 1000)
response.headers['changes-server-time'] = time_taken
# how much time did we spend waiting on the db
db_time_in_sec = sum([q.duration for q in get_debug_queries()])
db_timer_name = "changes_api_total_db_time_method_{}_class_{}".format(
request.method, self.__class__.__name__)
statsreporter.stats().log_timing(db_timer_name, db_time_in_sec * 1000)
response.headers['changes-server-db-time'] = db_time_in_sec
return response
def _find_and_retry_jobsteps(phase, implementation):
# phase.steps is ordered by date_started, so we retry the oldest jobsteps first
should_retry = [s for s in phase.steps if _should_retry_jobstep(s)]
if not should_retry:
return
already_retried = dict(
db.session.query(JobStep.node_id, func.count(JobStep.node_id)).filter(
JobStep.job == phase.job,
JobStep.replacement_id.isnot(None)).group_by(JobStep.node_id))
for step in should_retry:
# hard max on how many jobsteps we retry
if (sum(already_retried.itervalues()) >=
current_app.config['JOBSTEP_RETRY_MAX']):
break
# max on how many different failing machines we'll retry jobsteps for.
if (step.node_id not in already_retried and len(already_retried) >=
current_app.config['JOBSTEP_MACHINE_RETRY_MAX']):
break
newstep = implementation.create_replacement_jobstep(step)
if newstep:
statsreporter.stats().incr('jobstep_replaced')
# NB: node_id could be None if the jobstep failed before we got a node_id
already_retried[step.node_id] = already_retried.get(
step.node_id, 0) + 1
def get(self):
"""
New GET method that returns a priority sorted list of possible jobsteps
to allocate. The scheduler can then decide which ones it can actually
allocate and makes a POST request to mark these as such with Changes.
Args (in the form of a query string):
cluster (Optional[str]): The cluster to look for jobsteps in.
limit (int (default 200)): Maximum number of jobsteps to return.
"""
args = self.get_parser.parse_args()
cluster = args.cluster
limit = args.limit
with statsreporter.stats().timer('jobstep_allocate_get'):
available_allocations = self.find_next_jobsteps(limit, cluster)
jobstep_results = []
for jobstep in available_allocations:
jobplan, buildstep = JobPlan.get_build_step_for_job(
jobstep.job_id)
assert jobplan and buildstep
limits = buildstep.get_resource_limits()
req_cpus = limits.get('cpus', 4)
req_mem = limits.get('memory', 8 * 1024)
allocation_cmd = buildstep.get_allocation_command(jobstep)
jobstep_data = self.serialize(jobstep)
jobstep_data['project'] = self.serialize(jobstep.project)
jobstep_data['resources'] = {
'cpus': req_cpus,
'mem': req_mem,
}
jobstep_data['cmd'] = allocation_cmd
jobstep_results.append(jobstep_data)
return self.respond({'jobsteps': jobstep_results})
def post(self):
"""
Given a list of jobstep ids, returns the ids of those that should
be aborted. This is a POST only because we're sending large-ish
amounts of data--no state is changed by this call.
"""
args = json.loads(request.data)
try:
jobstep_ids = args['jobstep_ids']
except KeyError:
return error('Missing jobstep_ids attribute')
for id in jobstep_ids:
try:
UUID(id)
except ValueError:
err = "Invalid jobstep id sent to jobstep_needs_abort: %s"
logging.warning(err, id, exc_info=True)
return error(err % id)
if len(jobstep_ids) == 0:
return self.respond({'needs_abort': []})
with statsreporter.stats().timer('jobstep_needs_abort'):
finished = db.session.query(JobStep.id, JobStep.result,
JobStep.data).filter(
JobStep.status == Status.finished,
JobStep.id.in_(jobstep_ids),
).all()
needs_abort = []
for (step_id, result, data) in finished:
if result == Result.aborted or data.get('timed_out'):
needs_abort.append(step_id)
return self.respond({'needs_abort': needs_abort})
def get(self):
"""
GET method that returns a priority sorted list of possible jobsteps
to allocate. The scheduler can then decide which ones it can actually
allocate and makes a POST request to mark these as such with Changes.
Args (in the form of a query string):
cluster (Optional[str]): The cluster to look for jobsteps in.
limit (int (default 200)): Maximum number of jobsteps to return.
"""
args = self.get_parser.parse_args()
cluster = args.cluster
limit = args.limit
with statsreporter.stats().timer('jobstep_allocate_get'):
available_allocations = self.find_next_jobsteps(limit, cluster)
jobstep_results = self.serialize(available_allocations)
buildstep_for_job_id = {}
for jobstep, jobstep_data in zip(available_allocations, jobstep_results):
if jobstep.job_id not in buildstep_for_job_id:
buildstep_for_job_id[jobstep.job_id] = JobPlan.get_build_step_for_job(jobstep.job_id)[1]
buildstep = buildstep_for_job_id[jobstep.job_id]
limits = buildstep.get_resource_limits()
req_cpus = limits.get('cpus', 4)
req_mem = limits.get('memory', 8 * 1024)
allocation_cmd = buildstep.get_allocation_command(jobstep)
jobstep_data['project'] = jobstep.project
jobstep_data['resources'] = {
'cpus': req_cpus,
'mem': req_mem,
}
jobstep_data['cmd'] = allocation_cmd
return self.respond({'jobsteps': jobstep_results})
def _report_created(self):
"""Reports to monitoring that a new Task was created."""
statsreporter.stats().incr('new_task_created_' + self.task_name)
def verify_all_children(self):
task_list = list(
Task.query.filter(
Task.parent_id == self.task_id,
Task.status != Status.finished,
))
if not task_list:
return Status.finished
current_datetime = datetime.utcnow()
need_expire = set()
need_run = set()
has_pending = False
for task in task_list:
if self.needs_expired(task):
need_expire.add(task)
continue
has_pending = True
if self.needs_requeued(task) and 'kwargs' in task.data:
need_run.add(task)
if need_expire:
Task.query.filter(Task.id.in_(
[n.id for n in need_expire]), ).update(
{
Task.date_modified: current_datetime,
Task.date_finished: current_datetime,
Task.status: Status.finished,
Task.result: Result.aborted,
},
synchronize_session=False)
db.session.commit()
if need_run:
for task in need_run:
child_kwargs = task.data['kwargs'].copy()
child_kwargs['parent_task_id'] = task.parent_id.hex
child_kwargs['task_id'] = task.task_id.hex
queue.delay(task.task_name, kwargs=child_kwargs)
Task.query.filter(Task.id.in_([n.id for n in need_run]), ).update(
{
Task.date_modified: current_datetime,
},
synchronize_session=False)
db.session.commit()
for name, count in Counter(
(task.task_name for task in need_run)).iteritems():
statsreporter.stats().incr('task_revived_by_parent_' + name,
count)
if has_pending:
status = Status.in_progress
else:
status = Status.finished
return status
def sync_job_step(step_id):
"""
Polls a jenkins build for updates. May have sync_artifact children.
"""
step = JobStep.query.get(step_id)
if not step:
return
jobplan, implementation = JobPlan.get_build_step_for_job(
job_id=step.job_id)
# only synchronize if upstream hasn't suggested we're finished
if step.status != Status.finished:
implementation.update_step(step=step)
db.session.flush()
_sync_from_artifact_store(step)
if step.status == Status.finished:
_sync_artifacts_for_jobstep(step)
is_finished = (
step.status == Status.finished and
# make sure all child tasks (like sync_artifact) have also finished
sync_job_step.verify_all_children() == Status.finished)
if not is_finished:
default_timeout = current_app.config['DEFAULT_JOB_TIMEOUT_MIN']
if has_timed_out(step, jobplan, default_timeout=default_timeout):
old_status = step.status
step.data['timed_out'] = True
implementation.cancel_step(step=step)
# Not all implementations can actually cancel, but it's dead to us as of now
# so we mark it as finished.
step.status = Status.finished
step.date_finished = datetime.utcnow()
# Implementations default to marking canceled steps as aborted,
# but we're not canceling on good terms (it should be done by now)
# so we consider it a failure here.
#
# We check whether the step was marked as in_progress to make a best
# guess as to whether this is an infrastructure failure, or the
# repository under test is just taking too long. This won't be 100%
# reliable, but is probably good enough.
if old_status == Status.in_progress:
step.result = Result.failed
else:
step.result = Result.infra_failed
db.session.add(step)
job = step.job
try_create(
FailureReason, {
'step_id': step.id,
'job_id': job.id,
'build_id': job.build_id,
'project_id': job.project_id,
'reason': 'timeout'
})
db.session.flush()
statsreporter.stats().incr('job_step_timed_out')
# If we timeout something that isn't in progress, that's our fault, and we should know.
if old_status != Status.in_progress:
current_app.logger.warning(
"Timed out jobstep that wasn't in progress: %s (was %s)",
step.id, old_status)
raise sync_job_step.NotFinished
# Ignore any 'failures' if the build did not finish properly.
# NOTE(josiah): we might want to include "unknown" and "skipped" here as
# well, or have some named condition like "not meaningful_result(step.result)".
if step.result in (Result.aborted, Result.infra_failed):
_report_jobstep_result(step)
return
# Check for FailureReason objects generated by child jobs
failure_result = _result_from_failure_reasons(step)
if failure_result and failure_result != step.result:
step.result = failure_result
db.session.add(step)
db.session.commit()
if failure_result == Result.infra_failed:
_report_jobstep_result(step)
return
try:
record_coverage_stats(step)
except Exception:
current_app.logger.exception(
'Failing recording coverage stats for step %s', step.id)
# We need the start time of this step's phase to determine if we're part of
# the last phase. So, if date_started is empty, wait for sync_phase to catch
# up and try again.
if _expects_tests(jobplan) and not step.phase.date_started:
current_app.logger.warning(
"Phase[%s].date_started is missing. Retrying Step", step.phase.id)
# Reset result to unknown to reduce window where test might be incorrectly green.
# Set status to in_progress so that the next sync_job_step will fetch status from Jenkins again.
step.result = Result.unknown
step.status = Status.in_progress
raise sync_job_step.NotFinished
missing_tests = is_missing_tests(step, jobplan)
try_create(ItemStat,
where={
'item_id': step.id,
'name': 'tests_missing',
'value': int(missing_tests),
})
if missing_tests:
if step.result != Result.failed:
step.result = Result.failed
db.session.add(step)
try_create(
FailureReason, {
'step_id': step.id,
'job_id': step.job_id,
'build_id': step.job.build_id,
'project_id': step.project_id,
'reason': 'missing_tests'
})
db.session.commit()
db.session.flush()
if has_test_failures(step):
if step.result != Result.failed:
step.result = Result.failed
db.session.add(step)
try_create(
FailureReason, {
'step_id': step.id,
'job_id': step.job_id,
'build_id': step.job.build_id,
'project_id': step.project_id,
'reason': 'test_failures'
})
db.session.commit()
_report_jobstep_result(step)
def get(self):
args = self.get_parser.parse_args()
# This project index generation is a prerequisite for rendering
# the homepage and the admin page; worth tracking both for user
# experience and to keep an eye on database responsiveness.
with statsreporter.stats().timer('generate_project_index'):
queryset = Project.query
if args.query:
queryset = queryset.filter(
or_(
func.lower(Project.name).contains(args.query.lower()),
func.lower(Project.slug).contains(args.query.lower()),
), )
if args.status:
queryset = queryset.filter(
Project.status == ProjectStatus[args.status])
if args.sort == 'name':
queryset = queryset.order_by(Project.name.asc())
elif args.sort == 'date':
queryset = queryset.order_by(Project.date_created.asc())
plans = []
if args.fetch_extra:
queryset = queryset.options(
joinedload(Project.repository, innerjoin=True))
# fetch plans separately to avoid many lazy database fetches
plans_list = self.serialize(
list(
Plan.query.filter(
Plan.status == PlanStatus.active).options(
joinedload(Plan.steps))))
plans = defaultdict(list)
for p in plans_list:
plans[p['project_id']].append(p)
# we could use the option names whitelist from
# project_details.py
options_list = list(ProjectOption.query)
options_dict = defaultdict(dict)
for o in options_list:
options_dict[o.project_id][o.name] = o.value
project_list = list(queryset)
context = []
if project_list:
latest_build_results = get_latest_builds_query(
p.id for p in project_list)
projects_missing_passing_build = []
for build in latest_build_results:
if build.result != Result.passed:
projects_missing_passing_build.append(build.project_id)
if projects_missing_passing_build:
extra_passing_build_results = get_latest_builds_query(
projects_missing_passing_build,
result=Result.passed,
)
else:
extra_passing_build_results = []
# serialize as a group for more effective batching
serialized_builds = self.serialize(latest_build_results +
extra_passing_build_results)
serialized_latest_builds = serialized_builds[:len(
latest_build_results)]
serialized_extra_passing_builds = serialized_builds[
-len(extra_passing_build_results):]
latest_build_map = dict(
zip([b.project_id for b in latest_build_results],
serialized_latest_builds))
passing_build_map = {}
for build in latest_build_results:
if build.result == Result.passed:
passing_build_map[
build.project_id] = latest_build_map.get(
build.project_id)
else:
passing_build_map[build.project_id] = None
passing_build_map.update(
zip([b.project_id for b in extra_passing_build_results],
serialized_extra_passing_builds))
if args.fetch_extra:
repo_ids = set()
repos = []
for project in project_list:
if project.repository_id not in repo_ids:
repos.append(project.repository)
repo_ids.add(project.repository)
repo_dict = dict(
zip([repo.id for repo in repos],
self.serialize(repos)))
for project, data in zip(project_list,
self.serialize(project_list)):
data['lastBuild'] = latest_build_map.get(project.id)
data['lastPassingBuild'] = passing_build_map.get(
project.id)
if args.fetch_extra:
data['repository'] = repo_dict[project.repository_id]
data['options'] = options_dict[project.id]
# we have to use the serialized version of the id
data['plans'] = plans[data['id']]
context.append(data)
return self.respond(context, serialize=False)
def post_impl(self):
"""
Notify Changes of a newly created diff.
Depending on system configuration, this may create 0 or more new builds,
and the resulting response will be a list of those build objects.
"""
# we manually check for arg presence here so we can send a more specific
# error message to the user (rather than a plain 400)
args = self.parser.parse_args()
if not args.repository:
# No need to postback a comment for this
statsreporter.stats().incr("diffs_repository_not_found")
return error("Repository not found")
repository = args.repository
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
# no projects bound to repository
if not projects:
return self.respond([])
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'phabricator.diff-trigger',
])))
# Filter out projects that aren't configured to run builds off of diffs
# - Diff trigger disabled
# - No build plans
projects = [
p for p in projects
if options.get(p.id, '1') == '1' and get_build_plans(p)
]
if not projects:
return self.respond([])
statsreporter.stats().incr('diffs_posted_from_phabricator')
label = args.label[:128]
author = args.author
message = args.message
sha = args.sha
target = 'D%s' % args['phabricator.revisionID']
try:
identify_revision(repository, sha)
except MissingRevision:
# This may just be a broken request (which is why we respond with a 400) but
# it also might indicate Phabricator and Changes being out of sync somehow,
# so we err on the side of caution and log it as an error.
logging.error(
"Diff %s was posted for an unknown revision (%s, %s)", target,
sha, repository.url)
# We should postback since this can happen if a user diffs dependent revisions
statsreporter.stats().incr("diffs_missing_base_revision")
return self.postback_error(
"Unable to find base revision {revision} in {repo} on Changes. Some possible reasons:\n"
" - You may be working on multiple stacked diffs in your local repository.\n"
" {revision} only exists in your local copy. Changes thus cannot apply your patch\n"
" - If you are sure that's not the case, it's possible you applied your patch to an extremely\n"
" recent revision which Changes hasn't picked up yet. Retry in a minute\n"
.format(
revision=sha,
repo=repository.url,
),
target,
problems=['sha', 'repository'])
source_data = {
'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
'phabricator.diffID': args['phabricator.diffID'],
'phabricator.revisionID': args['phabricator.revisionID'],
'phabricator.revisionURL': args['phabricator.revisionURL'],
}
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=''.join(line.decode('utf-8') for line in args.patch_file),
)
db.session.add(patch)
source = Source(
patch=patch,
repository=repository,
revision_sha=sha,
data=source_data,
)
db.session.add(source)
phabricatordiff = try_create(
PhabricatorDiff, {
'diff_id': args['phabricator.diffID'],
'revision_id': args['phabricator.revisionID'],
'url': args['phabricator.revisionURL'],
'source': source,
})
if phabricatordiff is None:
logging.warning("Diff %s, Revision %s already exists",
args['phabricator.diffID'],
args['phabricator.revisionID'])
# No need to inform user about this explicitly
statsreporter.stats().incr("diffs_already_exists")
return error("Diff already exists within Changes")
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
# We already filtered out empty build plans
assert plan_list, ('No plans defined for project {}'.format(
project.slug))
try:
if not files_changed_should_trigger_project(
files_changed,
project,
project_options[project.id],
sha,
diff=patch.diff):
logging.info(
'No changed files matched project trigger for project %s',
project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified
pass
except ProjectConfigError:
logging.error(
'Project config for project %s is not in a valid format. Author is %s.',
project.slug,
author.name,
exc_info=True)
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
tag="phabricator",
))
# This is the counterpoint to the above 'diffs_posted_from_phabricator';
# at this point we've successfully processed the diff, so comparing this
# stat to the above should give us the phabricator diff failure rate.
statsreporter.stats().incr(
'diffs_successfully_processed_from_phabricator')
return self.respond(builds)
def __call__(self, **kwargs):
with statsreporter.stats().timer('task_duration_' + self.task_name):
with self.lock:
self._run(kwargs)
def report_response_status(r, *args, **kwargs):
statsreporter.stats().incr('jenkins_api_response_{}'.format(
r.status_code))
def post(self):
"""
Notify Changes of a newly created diff.
Depending on system configuration, this may create 0 or more new builds,
and the resulting response will be a list of those build objects.
"""
args = self.parser.parse_args()
repository = args.repository
if not args.repository:
return error("Repository not found")
projects = list(
Project.query.options(subqueryload_all('plans'), ).filter(
Project.status == ProjectStatus.active,
Project.repository_id == repository.id,
))
# no projects bound to repository
if not projects:
return self.respond([])
options = dict(
db.session.query(
ProjectOption.project_id, ProjectOption.value).filter(
ProjectOption.project_id.in_([p.id for p in projects]),
ProjectOption.name.in_([
'phabricator.diff-trigger',
])))
projects = [p for p in projects if options.get(p.id, '1') == '1']
if not projects:
return self.respond([])
statsreporter.stats().incr('diffs_posted_from_phabricator')
label = args.label[:128]
author = args.author
message = args.message
sha = args.sha
target = 'D{}'.format(args['phabricator.revisionID'])
try:
identify_revision(repository, sha)
except MissingRevision:
# This may just be a broken request (which is why we respond with a 400) but
# it also might indicate Phabricator and Changes being out of sync somehow,
# so we err on the side of caution and log it as an error.
logging.error(
"Diff %s was posted for an unknown revision (%s, %s)", target,
sha, repository.url)
return error("Unable to find commit %s in %s." %
(sha, repository.url),
problems=['sha', 'repository'])
source_data = {
'phabricator.buildTargetPHID': args['phabricator.buildTargetPHID'],
'phabricator.diffID': args['phabricator.diffID'],
'phabricator.revisionID': args['phabricator.revisionID'],
'phabricator.revisionURL': args['phabricator.revisionURL'],
}
patch = Patch(
repository=repository,
parent_revision_sha=sha,
diff=''.join(args.patch_file),
)
db.session.add(patch)
source = Source(
patch=patch,
repository=repository,
revision_sha=sha,
data=source_data,
)
db.session.add(source)
phabricatordiff = try_create(
PhabricatorDiff, {
'diff_id': args['phabricator.diffID'],
'revision_id': args['phabricator.revisionID'],
'url': args['phabricator.revisionURL'],
'source': source,
})
if phabricatordiff is None:
logging.error("Diff %s, Revision %s already exists",
args['phabricator.diffID'],
args['phabricator.revisionID'])
return error("Diff already exists within Changes")
project_options = ProjectOptionsHelper.get_options(
projects, ['build.file-whitelist'])
diff_parser = DiffParser(patch.diff)
files_changed = diff_parser.get_changed_files()
collection_id = uuid.uuid4()
builds = []
for project in projects:
plan_list = get_build_plans(project)
if not plan_list:
logging.warning('No plans defined for project %s',
project.slug)
continue
try:
if not files_changed_should_trigger_project(
files_changed,
project,
project_options[project.id],
sha,
diff=patch.diff):
logging.info(
'No changed files matched project trigger for project %s',
project.slug)
continue
except InvalidDiffError:
# ok, the build will fail and the user will be notified
pass
except ProjectConfigError:
logging.error(
'Project config for project %s is not in a valid format. Author is %s.',
project.slug,
author.name,
exc_info=True)
builds.append(
create_build(
project=project,
collection_id=collection_id,
sha=sha,
target=target,
label=label,
message=message,
author=author,
patch=patch,
tag="phabricator",
))
# This is the counterpoint to the above 'diffs_posted_from_phabricator';
# at this point we've successfully processed the diff, so comparing this
# stat to the above should give us the phabricator diff failure rate.
statsreporter.stats().incr(
'diffs_successfully_processed_from_phabricator')
return self.respond(builds)
def sync_build(build_id):
"""
Synchronizing the build happens continuously until all jobs have reported in
as finished or have failed/aborted.
This task is responsible for:
- Checking in with jobs
- Aborting/retrying them if they're beyond limits
- Aggregating the results from jobs into the build itself
"""
build = Build.query.get(build_id)
if not build:
return
if build.status == Status.finished:
return
all_jobs = list(Job.query.filter(
Job.build_id == build_id,
))
is_finished = sync_build.verify_all_children() == Status.finished
if any(p.status != Status.finished for p in all_jobs):
is_finished = False
prev_started = build.date_started
build.date_started = safe_agg(
min, (j.date_started for j in all_jobs if j.date_started))
# We want to report how long we waited for the build to start once and only once,
# so we do it at the transition from not started to started.
if not prev_started and build.date_started:
queued_time = build.date_started - build.date_created
statsreporter.stats().log_timing('build_start_latency', _timedelta_to_millis(queued_time))
if is_finished:
# If there are no jobs (or no jobs with a finished date) fall back to
# finishing now, since at this point, the build is done executing.
build.date_finished = safe_agg(
max, (j.date_finished for j in all_jobs if j.date_finished), datetime.utcnow())
else:
build.date_finished = None
if build.date_started and build.date_finished:
build.duration = _timedelta_to_millis(build.date_finished - build.date_started)
else:
build.duration = None
if any(j.result is Result.failed for j in all_jobs):
build.result = Result.failed
elif is_finished:
build.result = aggregate_result((j.result for j in all_jobs))
else:
build.result = Result.unknown
if is_finished:
build.status = Status.finished
else:
# ensure we dont set the status to finished unless it actually is
new_status = aggregate_status((j.status for j in all_jobs))
if new_status != Status.finished:
build.status = new_status
if is_finished:
build.date_decided = datetime.utcnow()
decided_latency = build.date_decided - build.date_finished
statsreporter.stats().log_timing('build_decided_latency', _timedelta_to_millis(decided_latency))
else:
build.date_decided = None
if db.session.is_modified(build):
build.date_modified = datetime.utcnow()
db.session.add(build)
db.session.commit()
if not is_finished:
raise sync_build.NotFinished
with statsreporter.stats().timer('build_stat_aggregation'):
try:
aggregate_build_stat(build, 'test_count')
aggregate_build_stat(build, 'test_duration')
aggregate_build_stat(build, 'test_failures')
aggregate_build_stat(build, 'test_rerun_count')
aggregate_build_stat(build, 'tests_missing')
aggregate_build_stat(build, 'lines_covered')
aggregate_build_stat(build, 'lines_uncovered')
aggregate_build_stat(build, 'diff_lines_covered')
aggregate_build_stat(build, 'diff_lines_uncovered')
except Exception:
current_app.logger.exception('Failing recording aggregate stats for build %s', build.id)
fire_signal.delay(
signal='build.finished',
kwargs={'build_id': build.id.hex},
)
queue.delay('update_project_stats', kwargs={
'project_id': build.project_id.hex,
}, countdown=1)
def create_build(project,
collection_id,
label,
target,
message,
author,
change=None,
patch=None,
cause=None,
source=None,
sha=None,
source_data=None,
tag=None,
snapshot_id=None,
no_snapshot=False):
assert sha or source
repository = project.repository
if source is None:
if patch:
source, _ = get_or_create(Source,
where={
'patch': patch,
},
defaults={
'repository': repository,
'revision_sha': sha,
'data': source_data or {},
})
else:
source, _ = get_or_create(Source,
where={
'repository': repository,
'patch': None,
'revision_sha': sha,
},
defaults={
'data': source_data or {},
})
statsreporter.stats().incr('new_api_build')
build = Build(
project=project,
project_id=project.id,
collection_id=collection_id,
source=source,
source_id=source.id if source else None,
status=Status.queued,
author=author,
author_id=author.id if author else None,
label=label,
target=target,
message=message,
cause=cause,
tags=[tag] if tag else [],
)
db.session.add(build)
db.session.commit()
execute_build(build=build,
snapshot_id=snapshot_id,
no_snapshot=no_snapshot)
return build
