def lookup_system_symbols(symbols, sdk_info=None, cpu_name=None):
"""Looks for system symbols in the configured system server if
enabled. If this failes or the server is disabled, `None` is
returned.
"""
if not options.get('symbolserver.enabled'):
return
url = '%s/lookup' % options.get('symbolserver.options')['url'].rstrip('/')
sess = Session()
symbol_query = {
'sdk_id': sdk_info_to_sdk_id(sdk_info),
'cpu_name': cpu_name,
'symbols': symbols,
}
attempts = 0
with sess:
while 1:
try:
rv = sess.post(url, json=symbol_query)
# If the symbols server does not know about the SDK at all
# it will report a 404 here. In that case just assume
# that we did not find a match and do not retry.
if rv.status_code == 404:
return None
rv.raise_for_status()
return rv.json()['symbols']
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact system symbol server', exc_info=True)
return
def _fetch_registry_url(relative_url):
if not settings.SENTRY_RELEASE_REGISTRY_BASEURL:
return {}
base_url = settings.SENTRY_RELEASE_REGISTRY_BASEURL.rstrip("/")
relative_url = relative_url.lstrip("/")
full_url = f"{base_url}/{relative_url}"
with metrics.timer("release_registry.fetch.duration",
tags={"url": relative_url},
sample_rate=1.0):
with Session() as session:
response = session.get(full_url, timeout=REQUEST_TIMEOUT)
response.raise_for_status()
return response.json()
def generate_chart(self,
style: ChartType,
data: Any,
upload: bool = True) -> Union[str, bytes]:
request_id = uuid4().hex
data = {
"requestId": request_id,
"style": style.value,
"data": data,
}
with Session() as session:
with sentry_sdk.start_span(
op="charts.chartcuterie.generate_chart",
description=type(self).__name__,
):
resp = session.request(
method="POST",
url=urljoin(self.service_url, "render"),
json=data,
)
if resp.status_code == 503 and settings.DEBUG:
logger.info(
"You may need to build the chartcuterie config using `yarn build-chartcuterie-config`"
)
if resp.status_code != 200:
raise RuntimeError(
f"Chartcuterie responded with {resp.status_code}: {resp.text}"
)
if not upload:
return resp.content
file_name = f"{request_id}.png"
with sentry_sdk.start_span(
op="charts.chartcuterie.upload",
description=type(self).__name__,
):
storage = get_storage(self.storage_options)
storage.save(file_name, BytesIO(resp.content))
url = absolute_uri(storage.url(file_name))
return url
def get_sdk_index():
value = cache.get(SDK_INDEX_CACHE_KEY)
if value is not None:
return value
base_url = settings.SENTRY_RELEASE_REGISTRY_BASEURL
if not base_url:
return {}
url = '%s/sdks' % (base_url, )
try:
with Session() as session:
response = session.get(url, timeout=1)
response.raise_for_status()
json = response.json()
except Exception:
logger.exception("Failed to fetch version index from release registry")
json = {}
cache.set(SDK_INDEX_CACHE_KEY, json, 3600)
return json
def open(self):
if self.session is None:
self.session = Session()
class SymbolicatorSession:
def __init__(self,
url=None,
sources=None,
project_id=None,
event_id=None,
timeout=None,
options=None):
self.url = url
self.project_id = project_id
self.event_id = event_id
self.sources = sources or []
self.options = options or None
self.timeout = timeout
self.session = None
def __enter__(self):
self.open()
return self
def __exit__(self, *args):
self.close()
def open(self):
if self.session is None:
self.session = Session()
def close(self):
if self.session is not None:
self.session.close()
self.session = None
def _ensure_open(self):
if not self.session:
raise RuntimeError("Session not opened")
def _process_response(self, json):
source_names = {
source["id"]: source.get("name")
for source in self.sources
}
source_names[INTERNAL_SOURCE_NAME] = "Sentry"
for module in json.get("modules") or ():
for candidate in module.get("candidates") or ():
if candidate.get("source"):
candidate["source_name"] = source_names.get(
candidate["source"])
return json
def _request(self, method, path, **kwargs):
self._ensure_open()
url = urljoin(self.url, path)
# required for load balancing
kwargs.setdefault("headers",
{})["x-sentry-project-id"] = self.project_id
kwargs.setdefault("headers", {})["x-sentry-event-id"] = self.event_id
attempts = 0
wait = 0.5
while True:
try:
with metrics.timer("events.symbolicator.session.request",
tags={"attempt": attempts}):
response = self.session.request(
method,
url,
timeout=settings.SYMBOLICATOR_POLL_TIMEOUT + 1,
**kwargs)
metrics.incr(
"events.symbolicator.status_code",
tags={
"status_code": response.status_code,
"project_id": self.project_id
},
)
if (method.lower() == "get" and path.startswith("requests/")
and response.status_code == 404):
# The symbolicator does not know this task. This is
# expected to happen when we're currently deploying
# symbolicator (which will clear all of its state). Re-send
# the symbolication task.
return None
if response.status_code in (502, 503):
raise ServiceUnavailable()
if response.ok:
json = response.json()
else:
json = {
"status": "failed",
"message": "internal server error"
}
return self._process_response(json)
except (OSError, RequestException) as e:
metrics.incr(
"events.symbolicator.request_error",
tags={
"exc":
".".join(
[e.__class__.__module__, e.__class__.__name__]),
"attempt":
attempts,
},
)
attempts += 1
# Any server error needs to be treated as a failure. We can
# retry a couple of times, but ultimately need to bail out.
#
# This can happen for any network failure.
if attempts > MAX_ATTEMPTS:
logger.error("Failed to contact symbolicator",
exc_info=True)
raise
time.sleep(wait)
wait *= 2.0
def _create_task(self, path, **kwargs):
params = {"timeout": self.timeout, "scope": self.project_id}
with metrics.timer("events.symbolicator.create_task",
tags={"path": path}):
return self._request(method="post",
path=path,
params=params,
**kwargs)
def symbolicate_stacktraces(self, stacktraces, modules, signal=None):
json = {
"sources": self.sources,
"options": self.options,
"stacktraces": stacktraces,
"modules": modules,
}
if signal:
json["signal"] = signal
return self._create_task("symbolicate", json=json)
def upload_minidump(self, minidump):
return self._create_task(
path="minidump",
data={
"sources": json.dumps(self.sources),
"options": json.dumps(self.options)
},
files={"upload_file_minidump": minidump},
)
def upload_applecrashreport(self, report):
return self._create_task(
path="applecrashreport",
data={
"sources": json.dumps(self.sources),
"options": json.dumps(self.options)
},
files={"apple_crash_report": report},
)
def query_task(self, task_id):
task_url = f"requests/{task_id}"
params = {
"timeout":
0, # Only wait when creating, but not when querying tasks
"scope": self.project_id,
}
with metrics.timer("events.symbolicator.query_task"):
return self._request("get", task_url, params=params)
def healthcheck(self):
return self._request("get", "healthcheck")
class SymbolicatorSession(object):
def __init__(self,
url=None,
sources=None,
project_id=None,
event_id=None,
timeout=None):
self.url = url
self.project_id = project_id
self.event_id = event_id
self.sources = sources or []
self.timeout = timeout
self.session = None
self._query_params = {"timeout": timeout, "scope": project_id}
def __enter__(self):
self.open()
return self
def __exit__(self, *args):
self.close()
def open(self):
if self.session is None:
self.session = Session()
def close(self):
if self.session is not None:
self.session.close()
self.session = None
def _ensure_open(self):
if not self.session:
raise RuntimeError("Session not opened")
def _request(self, method, path, **kwargs):
self._ensure_open()
url = urljoin(self.url, path)
# required for load balancing
kwargs.setdefault("headers",
{})["x-sentry-project-id"] = self.project_id
kwargs.setdefault("headers", {})["x-sentry-event-id"] = self.event_id
attempts = 0
wait = 0.5
while True:
try:
response = self.session.request(method, url, **kwargs)
metrics.incr(
"events.symbolicator.status_code",
tags={
"status_code": response.status_code,
"project_id": self.project_id
},
)
if (method.lower() == "get" and path.startswith("requests/")
and response.status_code == 404):
# The symbolicator does not know this task. This is
# expected to happen when we're currently deploying
# symbolicator (which will clear all of its state). Re-send
# the symbolication task.
return None
if response.status_code in (502, 503):
raise ServiceUnavailable()
if response.ok:
json = response.json()
else:
json = {
"status": "failed",
"message": "internal server error"
}
return json
except (IOError, RequestException):
attempts += 1
# Any server error needs to be treated as a failure. We can
# retry a couple of times, but ultimately need to bail out.
#
# This can happen for any network failure.
if attempts > MAX_ATTEMPTS:
logger.error("Failed to contact symbolicator",
exc_info=True)
raise
time.sleep(wait)
wait *= 2.0
def symbolicate_stacktraces(self, stacktraces, modules, signal=None):
json = {
"sources": self.sources,
"stacktraces": stacktraces,
"modules": modules
}
if signal:
json["signal"] = signal
return self._request("post",
"symbolicate",
params=self._query_params,
json=json)
def upload_minidump(self, minidump):
return self._request(
method="post",
path="minidump",
params=self._query_params,
data={"sources": json.dumps(self.sources)},
files={"upload_file_minidump": minidump},
)
def upload_applecrashreport(self, report):
return self._request(
method="post",
path="applecrashreport",
params=self._query_params,
data={"sources": json.dumps(self.sources)},
files={"apple_crash_report": report},
)
def query_task(self, task_id):
task_url = "requests/%s" % (task_id, )
return self._request("get", task_url, params=self._query_params)
def healthcheck(self):
return self._request("get", "healthcheck")
def run_symbolicator(stacktraces, modules, project, arch, signal,
request_id_cache_key):
internal_url_prefix = options.get('system.internal-url-prefix') \
or options.get('system.url-prefix')
assert internal_url_prefix
sentry_source_url = '%s%s' % (
internal_url_prefix.rstrip('/'),
reverse('sentry-api-0-dsym-files',
kwargs={
'organization_slug': project.organization.slug,
'project_slug': project.slug
}))
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
attempts = 0
wait = 0.5
with sess:
while 1:
try:
if request_id:
rv = _poll_symbolication_task(sess=sess,
base_url=base_url,
request_id=request_id)
else:
rv = _create_symbolication_task(
sess=sess,
base_url=base_url,
project_id=project_id,
sentry_source_url=sentry_source_url,
signal=signal,
stacktraces=stacktraces,
modules=modules)
metrics.incr('events.symbolicator.status.%s' % rv.status_code,
tags={'project_id': project_id})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response.%s' %
json['status'],
tags={'project_id': project_id})
if json['status'] == 'pending':
default_cache.set(request_id_cache_key, json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
elif json['status'] == 'completed':
default_cache.delete(request_id_cache_key)
return rv.json()
else:
logger.error("Unexpected status: %s", json['status'])
default_cache.delete(request_id_cache_key)
return
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator',
exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0
class SymbolicatorSession(object):
def __init__(self,
url=None,
sources=None,
project_id=None,
event_id=None,
timeout=None):
self.url = url
self.project_id = project_id
self.event_id = event_id
self.sources = sources or []
self.timeout = timeout
self.session = None
self._query_params = {'timeout': timeout, 'scope': project_id}
def __enter__(self):
self.open()
return self
def __exit__(self, *args):
self.close()
def open(self):
if self.session is None:
self.session = Session()
def close(self):
if self.session is not None:
self.session.close()
self.session = None
def _ensure_open(self):
if not self.session:
raise RuntimeError('Session not opened')
def _request(self, method, path, **kwargs):
self._ensure_open()
url = urljoin(self.url, path)
# required for load balancing
kwargs.setdefault('headers',
{})['x-sentry-project-id'] = self.project_id
kwargs.setdefault('headers', {})['x-sentry-event-id'] = self.event_id
attempts = 0
wait = 0.5
while True:
try:
response = self.session.request(method, url, **kwargs)
metrics.incr('events.symbolicator.status_code',
tags={
'status_code': response.status_code,
'project_id': self.project_id,
})
if (method.lower() == 'get' and path.startswith('requests/')
and response.status_code == 404):
# The symbolicator does not know this task. This is
# expected to happen when we're currently deploying
# symbolicator (which will clear all of its state). Re-send
# the symbolication task.
return None
if response.status_code == 503:
raise ServiceUnavailable()
response.raise_for_status()
json = response.json()
return json
except (IOError, RequestException):
attempts += 1
# Any server error needs to be treated as a failure. We can
# retry a couple of times, but ultimately need to bail out.
#
# This can happen for any network failure.
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator',
exc_info=True)
raise
time.sleep(wait)
wait *= 2.0
def symbolicate_stacktraces(self, stacktraces, modules, signal=None):
json = {
'sources': self.sources,
'stacktraces': stacktraces,
'modules': modules,
}
if signal:
json['signal'] = signal
return self._request('post',
'symbolicate',
params=self._query_params,
json=json)
def upload_minidump(self, minidump):
files = {'upload_file_minidump': minidump}
data = {
'sources': json.dumps(self.sources),
}
return self._request('post',
'minidump',
params=self._query_params,
data=data,
files=files)
def query_task(self, task_id):
task_url = 'requests/%s' % (task_id, )
return self._request('get', task_url, params=self._query_params)
def healthcheck(self):
return self._request('get', 'healthcheck')
def run_symbolicator(stacktraces, modules, project, arch, signal,
request_id_cache_key):
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
# Will be set lazily when a symbolicator request is fired
sources = None
attempts = 0
wait = 0.5
with sess:
while True:
try:
if request_id:
rv = _poll_symbolication_task(sess=sess,
base_url=base_url,
request_id=request_id)
else:
if sources is None:
sources = get_sources_for_project(project)
rv = _create_symbolication_task(sess=sess,
base_url=base_url,
project_id=project_id,
sources=sources,
signal=signal,
stacktraces=stacktraces,
modules=modules)
metrics.incr('events.symbolicator.status_code',
tags={
'status_code': rv.status_code,
'project_id': project_id,
})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response',
tags={
'response': json['status'],
'project_id': project_id,
})
if json['status'] == 'pending':
default_cache.set(request_id_cache_key, json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
elif json['status'] == 'completed':
default_cache.delete(request_id_cache_key)
return rv.json()
else:
logger.error("Unexpected status: %s", json['status'])
default_cache.delete(request_id_cache_key)
return
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator',
exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0
class SymbolicatorSession:
# used in x-sentry-worker-id http header
# to keep it static for celery worker process keep it as class attribute
_worker_id = None
def __init__(self,
url=None,
sources=None,
project_id=None,
event_id=None,
timeout=None,
options=None):
self.url = url
self.project_id = project_id
self.event_id = event_id
self.sources = sources or []
self.options = options or None
self.timeout = timeout
self.session = None
# Build some maps for use in ._process_response()
self.reverse_source_aliases = reverse_aliases_map(
settings.SENTRY_BUILTIN_SOURCES)
self.source_names = {
source["id"]: source.get("name", "unknown")
for source in self.sources
}
# Add a name for the special "sentry:project" source.
self.source_names[INTERNAL_SOURCE_NAME] = "Sentry"
# Add names for aliased sources.
for source in settings.SENTRY_BUILTIN_SOURCES.values():
if source.get("type") == "alias":
self.source_names[source["id"]] = source.get("name", "unknown")
# Remove sources that should be ignored. This leaves a few extra entries in the alias
# maps and source names maps, but that's fine. The orphaned entries in the maps will just
# never be used.
self.sources = filter_ignored_sources(self.sources,
self.reverse_source_aliases)
def __enter__(self):
self.open()
return self
def __exit__(self, *args):
self.close()
def open(self):
if self.session is None:
self.session = Session()
def close(self):
if self.session is not None:
self.session.close()
self.session = None
def _ensure_open(self):
if not self.session:
raise RuntimeError("Session not opened")
def _process_response(self, json):
"""Post-processes the JSON repsonse.
This modifies the candidates list from Symbolicator responses to undo aliased
sources, hide information about unknown sources and add names to sources rather then
just have their IDs.
"""
for module in json.get("modules") or ():
for candidate in module.get("candidates") or ():
# Reverse internal source aliases from the response.
source_id = candidate["source"]
original_source_id = self.reverse_source_aliases.get(source_id)
if original_source_id is not None:
candidate["source"] = original_source_id
source_id = original_source_id
# Add a "source_name" field to save the UI a lookup.
candidate["source_name"] = self.source_names.get(
source_id, "unknown")
redact_internal_sources(json)
return json
def _request(self, method, path, **kwargs):
self._ensure_open()
url = urljoin(self.url, path)
# required for load balancing
kwargs.setdefault("headers",
{})["x-sentry-project-id"] = self.project_id
kwargs.setdefault("headers", {})["x-sentry-event-id"] = self.event_id
kwargs.setdefault("headers",
{})["x-sentry-worker-id"] = self.get_worker_id()
attempts = 0
wait = 0.5
while True:
try:
with metrics.timer("events.symbolicator.session.request",
tags={"attempt": attempts}):
response = self.session.request(
method,
url,
timeout=settings.SYMBOLICATOR_POLL_TIMEOUT + 1,
**kwargs)
metrics.incr(
"events.symbolicator.status_code",
tags={
"status_code": response.status_code,
"project_id": self.project_id
},
)
if (method.lower() == "get" and path.startswith("requests/")
and response.status_code == 404):
# The symbolicator does not know this task. This is
# expected to happen when we're currently deploying
# symbolicator (which will clear all of its state). Re-send
# the symbolication task.
return None
if response.status_code in (502, 503):
raise ServiceUnavailable()
if response.ok:
json = response.json()
else:
json = {
"status": "failed",
"message": "internal server error"
}
return self._process_response(json)
except (OSError, RequestException) as e:
metrics.incr(
"events.symbolicator.request_error",
tags={
"exc":
".".join(
[e.__class__.__module__, e.__class__.__name__]),
"attempt":
attempts,
},
)
attempts += 1
# Any server error needs to be treated as a failure. We can
# retry a couple of times, but ultimately need to bail out.
#
# This can happen for any network failure.
if attempts > MAX_ATTEMPTS:
logger.error("Failed to contact symbolicator",
exc_info=True)
raise
time.sleep(wait)
wait *= 2.0
def _create_task(self, path, **kwargs):
params = {"timeout": self.timeout, "scope": self.project_id}
with metrics.timer(
"events.symbolicator.create_task",
tags={
"path": path,
"worker_id": self.get_worker_id()
},
):
return self._request(method="post",
path=path,
params=params,
**kwargs)
def symbolicate_stacktraces(self, stacktraces, modules, signal=None):
json = {
"sources": self.sources,
"options": self.options,
"stacktraces": stacktraces,
"modules": modules,
}
if signal:
json["signal"] = signal
return self._create_task("symbolicate", json=json)
def upload_minidump(self, minidump):
return self._create_task(
path="minidump",
data={
"sources": json.dumps(self.sources),
"options": json.dumps(self.options)
},
files={"upload_file_minidump": minidump},
)
def upload_applecrashreport(self, report):
return self._create_task(
path="applecrashreport",
data={
"sources": json.dumps(self.sources),
"options": json.dumps(self.options)
},
files={"apple_crash_report": report},
)
def query_task(self, task_id):
task_url = f"requests/{task_id}"
params = {
"timeout":
0, # Only wait when creating, but not when querying tasks
"scope": self.project_id,
}
with metrics.timer("events.symbolicator.query_task",
tags={"worker_id": self.get_worker_id()}):
return self._request("get", task_url, params=params)
def healthcheck(self):
return self._request("get", "healthcheck")
@classmethod
def get_worker_id(cls):
# as class attribute to keep it static for life of process
if cls._worker_id is None:
# %5000 to reduce cardinality of metrics tagging with worker id
cls._worker_id = str(uuid.uuid4().int % 5000)
return cls._worker_id
