def get_attachments(cache_key, event):
"""
Computes a list of attachments that should be stored.
This method checks whether event attachments are available and sends them to
the blob store. There is special handling for crash reports which may
contain unstripped PII. If the project or organization is configured to
limit the amount of crash reports per group, the number of stored crashes is
limited.
:param cache_key: The cache key at which the event payload is stored in the
cache. This is used to retrieve attachments.
:param event: The event model instance.
"""
filtered = []
if cache_key is None:
return filtered
project = event.project
if not features.has("organizations:event-attachments",
project.organization,
actor=None):
return filtered
attachments = list(attachment_cache.get(cache_key))
if not attachments:
return filtered
# The setting is both an organization and project setting. The project
# setting strictly overrides the organization setting, unless set to the
# default.
max_crashreports = get_max_crashreports(project)
if not max_crashreports:
max_crashreports = get_max_crashreports(project.organization)
# The number of crash reports is cached per group
crashreports_key = get_crashreport_key(event.group_id)
# Only fetch the number of stored crash reports if there is a crash report
# in the list of attachments. Otherwise, we won't require this number.
if any(attachment.type in CRASH_REPORT_TYPES
for attachment in attachments):
cached_reports = get_stored_crashreports(crashreports_key, event,
max_crashreports)
else:
cached_reports = 0
stored_reports = cached_reports
for attachment in attachments:
# If the attachment is a crash report (e.g. minidump), we need to honor
# the store_crash_reports setting. Otherwise, we assume that the client
# has already verified PII and just store the attachment.
if attachment.type in CRASH_REPORT_TYPES:
if crashreports_exceeded(stored_reports, max_crashreports):
continue
stored_reports += 1
filtered.append(attachment)
# Check if we have exceeded the stored crash reports count. If so, we
# persist the current maximum (not the actual number!) into the cache. Next
# time when loading from the cache, we will validate that this number has
# not changed, or otherwise re-fetch from the database.
if crashreports_exceeded(
stored_reports,
max_crashreports) and stored_reports > cached_reports:
cache.set(crashreports_key, max_crashreports, CRASH_REPORT_TIMEOUT)
return filtered
def filter_attachments_for_group(attachments, job):
"""
Removes crash reports exceeding the group-limit.
If the project or organization is configured to limit the amount of crash
reports per group, the number of stored crashes is limited. This requires
`event.group` to be set.
Emits one outcome per removed attachment.
:param attachments: The full list of attachments to filter.
:param job: The job context container.
"""
if not attachments:
return attachments
event = job["event"]
project = event.project
# The setting is both an organization and project setting. The project
# setting strictly overrides the organization setting, unless set to the
# default.
max_crashreports = get_max_crashreports(project, allow_none=True)
if max_crashreports is None:
max_crashreports = get_max_crashreports(project.organization)
# The number of crash reports is cached per group
crashreports_key = get_crashreport_key(event.group_id)
# Only fetch the number of stored crash reports if there is a crash report
# in the list of attachments. Otherwise, we won't require this number.
if any(attachment.type in CRASH_REPORT_TYPES for attachment in attachments):
cached_reports = get_stored_crashreports(crashreports_key, event, max_crashreports)
else:
cached_reports = 0
stored_reports = cached_reports
filtered = []
refund_quantity = 0
for attachment in attachments:
# If the attachment is a crash report (e.g. minidump), we need to honor
# the store_crash_reports setting. Otherwise, we assume that the client
# has already verified PII and just store the attachment.
if attachment.type in CRASH_REPORT_TYPES:
if crashreports_exceeded(stored_reports, max_crashreports):
# Indicate that the crash report has been removed due to a limit
# on the maximum number of crash reports. If this flag is True,
# it indicates that there are *other* events in the same group
# that store a crash report. This flag will therefore *not* be
# set if storage of crash reports is completely disabled.
if max_crashreports > 0:
job["data"]["metadata"]["stripped_crash"] = True
track_outcome(
org_id=event.project.organization_id,
project_id=job["project_id"],
key_id=job["key_id"],
outcome=Outcome.FILTERED,
reason=FilterStatKeys.CRASH_REPORT_LIMIT,
timestamp=to_datetime(job["start_time"]),
event_id=event.event_id,
category=DataCategory.ATTACHMENT,
quantity=attachment.size,
)
# Quotas are counted with at least ``1`` for attachments.
refund_quantity += attachment.size or 1
continue
stored_reports += 1
filtered.append(attachment)
# Check if we have exceeded the stored crash reports count. If so, we
# persist the current maximum (not the actual number!) into the cache. Next
# time when loading from the cache, we will validate that this number has
# not changed, or otherwise re-fetch from the database.
if crashreports_exceeded(stored_reports, max_crashreports) and stored_reports > cached_reports:
cache.set(crashreports_key, max_crashreports, CRASH_REPORT_TIMEOUT)
if refund_quantity:
quotas.refund(
project,
key=job["project_key"],
timestamp=job["start_time"],
category=DataCategory.ATTACHMENT,
quantity=refund_quantity,
)
return filtered
