def update_config_cache(generate, organization_id=None, project_id=None, update_reason=None):
"""
Update the Redis cache for the Relay projectconfig. This task is invoked
whenever a project/org option has been saved or smart quotas potentially
caused a change in projectconfig.
Either organization_id or project_id has to be provided.
:param organization_id: The organization for which to invalidate configs.
:param project_id: The project for which to invalidate configs.
:param generate: If `True`, caches will be eagerly regenerated, not only
invalidated.
"""
from sentry.models import Project
from sentry.relay import projectconfig_cache
from sentry.relay.config import get_project_config
# Delete key before generating configs such that we never have an outdated
# but valid cache.
#
# If this was running at the end of the task, it would be more effective
# against bursts of updates, but introduces a different race where an
# outdated cache may be used.
debounce_key = _get_schedule_debounce_key(project_id, organization_id)
cache.delete(debounce_key)
if project_id:
projects = [Project.objects.get_from_cache(id=project_id)]
elif organization_id:
# XXX(markus): I feel like we should be able to cache this but I don't
# want to add another method to src/sentry/db/models/manager.py
projects = Project.objects.filter(organization_id=organization_id)
if generate:
project_keys = {}
for key in ProjectKey.objects.filter(project_id__in=[project.id for project in projects]):
project_keys.setdefault(key.project_id, []).append(key)
project_configs = {}
for project in projects:
project_config = get_project_config(
project, project_keys=project_keys.get(project.id, []), full_config=True
)
project_configs[project.id] = project_config.to_dict()
projectconfig_cache.set_many(project_configs)
else:
projectconfig_cache.delete_many([project.id for project in projects])
metrics.incr(
"relay.projectconfig_cache.done",
tags={"generate": generate, "update_reason": update_reason},
)
def _post(self, request):
relay = request.relay
assert relay is not None # should be provided during Authentication
full_config_requested = request.relay_request_data.get("fullConfig")
if full_config_requested and not relay.is_internal:
return Response("Relay unauthorized for full config information",
403)
with Hub.current.start_span(op="relay_fetch_projects"):
project_ids = set(request.relay_request_data.get("projects") or ())
if project_ids:
with metrics.timer(
"relay_project_configs.fetching_projects.duration"):
projects = {
p.id: p
for p in Project.objects.get_many_from_cache(
project_ids)
}
else:
projects = {}
with Hub.current.start_span(op="relay_fetch_orgs"):
# Preload all organizations and their options to prevent repeated
# database access when computing the project configuration.
org_ids = set(project.organization_id
for project in six.itervalues(projects))
if org_ids:
with metrics.timer(
"relay_project_configs.fetching_orgs.duration"):
orgs = Organization.objects.get_many_from_cache(org_ids)
orgs = {
o.id: o
for o in orgs if request.relay.has_org_access(o)
}
else:
orgs = {}
org_options = {
i: OrganizationOption.objects.get_all_values(i)
for i in six.iterkeys(orgs)
}
with Hub.current.start_span(op="relay_fetch_keys"):
project_keys = {}
for key in ProjectKey.objects.filter(project_id__in=project_ids):
project_keys.setdefault(key.project_id, []).append(key)
metrics.timing("relay_project_configs.projects_requested",
len(project_ids))
metrics.timing("relay_project_configs.projects_fetched", len(projects))
metrics.timing("relay_project_configs.orgs_fetched", len(orgs))
configs = {}
for project_id in project_ids:
configs[six.text_type(project_id)] = {"disabled": True}
project = projects.get(int(project_id))
if project is None:
continue
organization = orgs.get(project.organization_id)
if organization is None:
continue
# Try to prevent organization from being fetched again in quotas.
project.organization = organization
project._organization_cache = organization
with Hub.current.start_span(op="get_config"):
with metrics.timer(
"relay_project_configs.get_config.duration"):
project_config = config.get_project_config(
project,
org_options=org_options.get(organization.id) or {},
full_config=full_config_requested,
project_keys=project_keys.get(project.id) or [],
)
configs[six.text_type(project_id)] = project_config.to_dict()
if full_config_requested:
projectconfig_cache.set_many(configs)
return Response({"configs": configs}, status=200)
def _post_by_key(self, request, full_config_requested):
public_keys = request.relay_request_data.get("publicKeys")
public_keys = set(public_keys or ())
project_keys = {} # type: dict[str, ProjectKey]
project_ids = set() # type: set[int]
with start_span(op="relay_fetch_keys"):
with metrics.timer("relay_project_configs.fetching_keys.duration"):
for key in ProjectKey.objects.get_many_from_cache(
public_keys, key="public_key"):
if key.status != ProjectKeyStatus.ACTIVE:
continue
project_keys[key.public_key] = key
project_ids.add(key.project_id)
projects = {} # type: dict[int, Project]
organization_ids = set() # type: set[int]
with start_span(op="relay_fetch_projects"):
with metrics.timer(
"relay_project_configs.fetching_projects.duration"):
for project in Project.objects.get_many_from_cache(
project_ids):
projects[project.id] = project
organization_ids.add(project.organization_id)
# Preload all organizations and their options to prevent repeated
# database access when computing the project configuration.
orgs = {} # type: dict[int, Organization]
with start_span(op="relay_fetch_orgs"):
with metrics.timer("relay_project_configs.fetching_orgs.duration"):
for org in Organization.objects.get_many_from_cache(
organization_ids):
if request.relay.has_org_access(org):
orgs[org.id] = org
with start_span(op="relay_fetch_org_options"):
with metrics.timer(
"relay_project_configs.fetching_org_options.duration"):
for org_id in orgs:
OrganizationOption.objects.get_all_values(org_id)
metrics.timing("relay_project_configs.projects_requested",
len(project_ids))
metrics.timing("relay_project_configs.projects_fetched", len(projects))
metrics.timing("relay_project_configs.orgs_fetched", len(orgs))
configs = {}
for public_key in public_keys:
configs[public_key] = {"disabled": True}
key = project_keys.get(public_key)
if key is None:
continue
project = projects.get(key.project_id)
if project is None:
continue
organization = orgs.get(project.organization_id)
if organization is None:
continue
# Try to prevent organization from being fetched again in quotas.
project.organization = organization
project._organization_cache = organization
with Hub.current.start_span(op="get_config"):
with metrics.timer(
"relay_project_configs.get_config.duration"):
project_config = config.get_project_config(
project,
full_config=full_config_requested,
project_keys=[key],
)
configs[public_key] = project_config.to_dict()
if full_config_requested:
projectconfig_cache.set_many(configs)
return Response({"configs": configs}, status=200)
def _post_by_project(self, request, full_config_requested):
project_ids = set(request.relay_request_data.get("projects") or ())
with start_span(op="relay_fetch_projects"):
if project_ids:
with metrics.timer(
"relay_project_configs.fetching_projects.duration"):
projects = {
p.id: p
for p in Project.objects.get_many_from_cache(
project_ids)
}
else:
projects = {}
with start_span(op="relay_fetch_orgs"):
# Preload all organizations and their options to prevent repeated
# database access when computing the project configuration.
org_ids = {
project.organization_id
for project in projects.values()
}
if org_ids:
with metrics.timer(
"relay_project_configs.fetching_orgs.duration"):
orgs = Organization.objects.get_many_from_cache(org_ids)
orgs = {
o.id: o
for o in orgs if request.relay.has_org_access(o)
}
else:
orgs = {}
with metrics.timer(
"relay_project_configs.fetching_org_options.duration"):
for org_id in orgs.keys():
OrganizationOption.objects.get_all_values(org_id)
with start_span(op="relay_fetch_keys"):
project_keys = {}
for key in ProjectKey.objects.filter(project_id__in=project_ids):
project_keys.setdefault(key.project_id, []).append(key)
metrics.timing("relay_project_configs.projects_requested",
len(project_ids))
metrics.timing("relay_project_configs.projects_fetched", len(projects))
metrics.timing("relay_project_configs.orgs_fetched", len(orgs))
configs = {}
for project_id in project_ids:
configs[str(project_id)] = {"disabled": True}
project = projects.get(int(project_id))
if project is None:
continue
organization = orgs.get(project.organization_id)
if organization is None:
continue
# Try to prevent organization from being fetched again in quotas.
project.organization = organization
project._organization_cache = organization
with start_span(op="get_config"):
with metrics.timer(
"relay_project_configs.get_config.duration"):
project_config = config.get_project_config(
project,
full_config=full_config_requested,
project_keys=project_keys.get(project.id) or [],
)
configs[str(project_id)] = project_config.to_dict()
if full_config_requested:
projectconfig_cache.set_many(configs)
return Response({"configs": configs}, status=200)
def update_config_cache(generate,
organization_id=None,
project_id=None,
public_key=None,
update_reason=None):
"""
Update the Redis cache for the Relay projectconfig. This task is invoked
whenever a project/org option has been saved or smart quotas potentially
caused a change in projectconfig.
Either organization_id or project_id has to be provided.
:param organization_id: The organization for which to invalidate configs.
:param project_id: The project for which to invalidate configs.
:param generate: If `True`, caches will be eagerly regenerated, not only
invalidated.
"""
from sentry.models import Project, ProjectKey, ProjectKeyStatus
from sentry.relay import projectconfig_cache
from sentry.relay.config import get_project_config
if project_id:
set_current_event_project(project_id)
if organization_id:
# Cannot use bind_organization_context here because we do not have a
# model and don't want to fetch one
sentry_sdk.set_tag("organization_id", organization_id)
if public_key:
sentry_sdk.set_tag("public_key", public_key)
sentry_sdk.set_tag("update_reason", update_reason)
sentry_sdk.set_tag("generate", generate)
# Delete key before generating configs such that we never have an outdated
# but valid cache.
#
# If this was running at the end of the task, it would be more effective
# against bursts of updates, but introduces a different race where an
# outdated cache may be used.
projectconfig_debounce_cache.mark_task_done(public_key, project_id,
organization_id)
if organization_id:
projects = list(
Project.objects.filter(organization_id=organization_id))
keys = list(ProjectKey.objects.filter(project__in=projects))
elif project_id:
projects = [Project.objects.get(id=project_id)]
keys = list(ProjectKey.objects.filter(project__in=projects))
elif public_key:
try:
keys = [ProjectKey.objects.get(public_key=public_key)]
except ProjectKey.DoesNotExist:
# In this particular case, where a project key got deleted and
# triggered an update, we at least know the public key that needs
# to be deleted from cache.
#
# In other similar cases, like an org being deleted, we potentially
# cannot find any keys anymore, so we don't know which cache keys
# to delete.
projectconfig_cache.delete_many([public_key])
return
else:
assert False
if generate:
config_cache = {}
for key in keys:
if key.status != ProjectKeyStatus.ACTIVE:
project_config = {"disabled": True}
else:
project_config = get_project_config(
key.project, project_keys=[key],
full_config=True).to_dict()
config_cache[key.public_key] = project_config
projectconfig_cache.set_many(config_cache)
else:
cache_keys_to_delete = []
for key in keys:
cache_keys_to_delete.append(key.public_key)
projectconfig_cache.delete_many(cache_keys_to_delete)
def update_config_cache(generate,
organization_id=None,
project_id=None,
update_reason=None):
"""
Update the Redis cache for the Relay projectconfig. This task is invoked
whenever a project/org option has been saved or smart quotas potentially
caused a change in projectconfig.
Either organization_id or project_id has to be provided.
:param organization_id: The organization for which to invalidate configs.
:param project_id: The project for which to invalidate configs.
:param generate: If `True`, caches will be eagerly regenerated, not only
invalidated.
"""
from sentry.models import Project, ProjectKey, ProjectKeyStatus
from sentry.relay import projectconfig_cache
from sentry.relay.config import get_project_config
if project_id:
set_current_event_project(project_id)
if organization_id:
# Cannot use bind_organization_context here because we do not have a
# model and don't want to fetch one
sentry_sdk.set_tag("organization_id", organization_id)
sentry_sdk.set_tag("update_reason", update_reason)
sentry_sdk.set_tag("generate", generate)
# Delete key before generating configs such that we never have an outdated
# but valid cache.
#
# If this was running at the end of the task, it would be more effective
# against bursts of updates, but introduces a different race where an
# outdated cache may be used.
projectconfig_debounce_cache.mark_task_done(project_id, organization_id)
if project_id:
projects = [Project.objects.get_from_cache(id=project_id)]
elif organization_id:
# XXX(markus): I feel like we should be able to cache this but I don't
# want to add another method to src/sentry/db/models/manager.py
projects = Project.objects.filter(organization_id=organization_id)
project_keys = {}
for key in ProjectKey.objects.filter(
project_id__in=[project.id for project in projects]):
project_keys.setdefault(key.project_id, []).append(key)
if generate:
config_cache = {}
for project in projects:
project_config = get_project_config(project,
project_keys=project_keys.get(
project.id, []),
full_config=True)
config_cache[project.id] = project_config.to_dict()
for key in project_keys.get(project.id) or ():
# XXX(markus): This is currently the cleanest way to get only
# state for a single projectkey (considering quotas and
# everything)
if key.status != ProjectKeyStatus.ACTIVE:
continue
project_config = get_project_config(project,
project_keys=[key],
full_config=True)
config_cache[key.public_key] = project_config.to_dict()
projectconfig_cache.set_many(config_cache)
else:
cache_keys_to_delete = []
for project in projects:
cache_keys_to_delete.append(project.id)
for key in project_keys.get(project.id) or ():
cache_keys_to_delete.append(key.public_key)
projectconfig_cache.delete_many(cache_keys_to_delete)
metrics.incr(
"relay.projectconfig_cache.done",
tags={
"generate": generate,
"update_reason": update_reason
},
)
