def _scan_for_suspect_projects() -> None:
suspect_projects = set()
for project_id in realtime_metrics.projects():
suspect_projects.add(project_id)
update_lpq_eligibility(project_id).apply_async()
# Prune projects we definitely know shouldn't be in the queue any more.
# `update_lpq_eligibility` should handle removing suspect projects from the list if it turns
# out they need to be evicted.
current_lpq_projects = realtime_metrics.get_lpq_projects() or set()
deleted_projects = current_lpq_projects.difference(suspect_projects)
if len(deleted_projects) == 0:
return
realtime_metrics.remove_projects_from_lpq(deleted_projects)
for project_id in deleted_projects:
# TODO: add metrics!
logger.warning("Moved project out of symbolicator's low priority queue: %s", project_id)
def _scan_for_suspect_projects() -> None:
suspect_projects = set()
now = int(time.time())
for project_id in realtime_metrics.projects():
suspect_projects.add(project_id)
update_lpq_eligibility.delay(project_id=project_id, cutoff=now)
# Prune projects we definitely know shouldn't be in the queue any more.
# `update_lpq_eligibility` should handle removing suspect projects from the list if it turns
# out they need to be evicted.
current_lpq_projects = realtime_metrics.get_lpq_projects() or set()
expired_projects = current_lpq_projects.difference(suspect_projects)
if not expired_projects:
return
realtime_metrics.remove_projects_from_lpq(expired_projects)
for project_id in expired_projects:
_report_change(project_id=project_id, change="removed", reason="no metrics")
def _scan_for_suspect_projects() -> None:
suspect_projects = set()
now = int(time.time())
for project_id in realtime_metrics.projects():
suspect_projects.add(project_id)
update_lpq_eligibility.delay(project_id=project_id, cutoff=now)
# Prune projects we definitely know shouldn't be in the queue any more.
# `update_lpq_eligibility` should handle removing suspect projects from the list if it turns
# out they need to be evicted.
current_lpq_projects = realtime_metrics.get_lpq_projects() or set()
expired_projects = current_lpq_projects.difference(suspect_projects)
if not expired_projects:
return
realtime_metrics.remove_projects_from_lpq(expired_projects)
for project_id in expired_projects:
# TODO: add metrics!
logger.warning("Moved project out of symbolicator's low priority queue: %s", project_id)
def _update_lpq_eligibility(project_id: int) -> None:
counts = realtime_metrics.get_counts_for_project(project_id)
durations = realtime_metrics.get_durations_for_project(project_id)
is_eligible = calculation_magic(counts, durations)
if is_eligible:
was_added = realtime_metrics.add_project_to_lpq(project_id)
if was_added:
logger.warning("Moved project to symbolicator's low priority queue: %s", project_id)
elif not is_eligible:
was_removed = realtime_metrics.remove_projects_from_lpq({project_id})
if was_removed:
logger.warning("Moved project out of symbolicator's low priority queue: %s", project_id)
def _update_lpq_eligibility(project_id: int, cutoff: int) -> None:
# TODO: It may be a good idea to figure out how to debounce especially if this is
# executing more than 10s after cutoff.
counts = realtime_metrics.get_counts_for_project(project_id, cutoff)
durations = realtime_metrics.get_durations_for_project(project_id, cutoff)
is_eligible = calculation_magic(counts, durations)
if is_eligible:
was_added = realtime_metrics.add_project_to_lpq(project_id)
if was_added:
logger.warning("Moved project to symbolicator's low priority queue: %s", project_id)
elif not is_eligible:
was_removed = realtime_metrics.remove_projects_from_lpq({project_id})
if was_removed:
logger.warning("Moved project out of symbolicator's low priority queue: %s", project_id)
def _update_lpq_eligibility(project_id: int, cutoff: int) -> None:
# TODO: It may be a good idea to figure out how to debounce especially if this is
# executing more than 10s after cutoff.
event_counts = realtime_metrics.get_counts_for_project(project_id, cutoff)
durations = realtime_metrics.get_durations_for_project(project_id, cutoff)
excessive_rate = excessive_event_rate(project_id, event_counts)
excessive_duration = excessive_event_duration(project_id, durations)
if excessive_rate or excessive_duration:
was_added = realtime_metrics.add_project_to_lpq(project_id)
if was_added:
reason = "rate" if excessive_rate else "duration"
if excessive_rate and excessive_duration:
reason = "rate-duration"
_report_change(project_id=project_id, change="added", reason=reason)
else:
was_removed = realtime_metrics.remove_projects_from_lpq({project_id})
if was_removed:
_report_change(project_id=project_id, change="removed", reason="ineligible")