def lookup_system_symbols(symbols, sdk_info=None, cpu_name=None): """Looks for system symbols in the configured system server if enabled. If this failes or the server is disabled, `None` is returned. """ if not options.get('symbolserver.enabled'): return url = '%s/lookup' % options.get('symbolserver.options')['url'].rstrip('/') sess = Session() symbol_query = { 'sdk_id': sdk_info_to_sdk_id(sdk_info), 'cpu_name': cpu_name, 'symbols': symbols, } attempts = 0 with sess: while 1: try: rv = sess.post(url, json=symbol_query) # If the symbols server does not know about the SDK at all # it will report a 404 here. In that case just assume # that we did not find a match and do not retry. if rv.status_code == 404: return None rv.raise_for_status() return rv.json()['symbols'] except (IOError, RequestException): attempts += 1 if attempts > MAX_ATTEMPTS: logger.error('Failed to contact system symbol server', exc_info=True) return
def _fetch_registry_url(relative_url): if not settings.SENTRY_RELEASE_REGISTRY_BASEURL: return {} base_url = settings.SENTRY_RELEASE_REGISTRY_BASEURL.rstrip("/") relative_url = relative_url.lstrip("/") full_url = f"{base_url}/{relative_url}" with metrics.timer("release_registry.fetch.duration", tags={"url": relative_url}, sample_rate=1.0): with Session() as session: response = session.get(full_url, timeout=REQUEST_TIMEOUT) response.raise_for_status() return response.json()
def generate_chart(self, style: ChartType, data: Any, upload: bool = True) -> Union[str, bytes]: request_id = uuid4().hex data = { "requestId": request_id, "style": style.value, "data": data, } with Session() as session: with sentry_sdk.start_span( op="charts.chartcuterie.generate_chart", description=type(self).__name__, ): resp = session.request( method="POST", url=urljoin(self.service_url, "render"), json=data, ) if resp.status_code == 503 and settings.DEBUG: logger.info( "You may need to build the chartcuterie config using `yarn build-chartcuterie-config`" ) if resp.status_code != 200: raise RuntimeError( f"Chartcuterie responded with {resp.status_code}: {resp.text}" ) if not upload: return resp.content file_name = f"{request_id}.png" with sentry_sdk.start_span( op="charts.chartcuterie.upload", description=type(self).__name__, ): storage = get_storage(self.storage_options) storage.save(file_name, BytesIO(resp.content)) url = absolute_uri(storage.url(file_name)) return url
def get_sdk_index(): value = cache.get(SDK_INDEX_CACHE_KEY) if value is not None: return value base_url = settings.SENTRY_RELEASE_REGISTRY_BASEURL if not base_url: return {} url = '%s/sdks' % (base_url, ) try: with Session() as session: response = session.get(url, timeout=1) response.raise_for_status() json = response.json() except Exception: logger.exception("Failed to fetch version index from release registry") json = {} cache.set(SDK_INDEX_CACHE_KEY, json, 3600) return json
def open(self): if self.session is None: self.session = Session()
class SymbolicatorSession: def __init__(self, url=None, sources=None, project_id=None, event_id=None, timeout=None, options=None): self.url = url self.project_id = project_id self.event_id = event_id self.sources = sources or [] self.options = options or None self.timeout = timeout self.session = None def __enter__(self): self.open() return self def __exit__(self, *args): self.close() def open(self): if self.session is None: self.session = Session() def close(self): if self.session is not None: self.session.close() self.session = None def _ensure_open(self): if not self.session: raise RuntimeError("Session not opened") def _process_response(self, json): source_names = { source["id"]: source.get("name") for source in self.sources } source_names[INTERNAL_SOURCE_NAME] = "Sentry" for module in json.get("modules") or (): for candidate in module.get("candidates") or (): if candidate.get("source"): candidate["source_name"] = source_names.get( candidate["source"]) return json def _request(self, method, path, **kwargs): self._ensure_open() url = urljoin(self.url, path) # required for load balancing kwargs.setdefault("headers", {})["x-sentry-project-id"] = self.project_id kwargs.setdefault("headers", {})["x-sentry-event-id"] = self.event_id attempts = 0 wait = 0.5 while True: try: with metrics.timer("events.symbolicator.session.request", tags={"attempt": attempts}): response = self.session.request( method, url, timeout=settings.SYMBOLICATOR_POLL_TIMEOUT + 1, **kwargs) metrics.incr( "events.symbolicator.status_code", tags={ "status_code": response.status_code, "project_id": self.project_id }, ) if (method.lower() == "get" and path.startswith("requests/") and response.status_code == 404): # The symbolicator does not know this task. This is # expected to happen when we're currently deploying # symbolicator (which will clear all of its state). Re-send # the symbolication task. return None if response.status_code in (502, 503): raise ServiceUnavailable() if response.ok: json = response.json() else: json = { "status": "failed", "message": "internal server error" } return self._process_response(json) except (OSError, RequestException) as e: metrics.incr( "events.symbolicator.request_error", tags={ "exc": ".".join( [e.__class__.__module__, e.__class__.__name__]), "attempt": attempts, }, ) attempts += 1 # Any server error needs to be treated as a failure. We can # retry a couple of times, but ultimately need to bail out. # # This can happen for any network failure. if attempts > MAX_ATTEMPTS: logger.error("Failed to contact symbolicator", exc_info=True) raise time.sleep(wait) wait *= 2.0 def _create_task(self, path, **kwargs): params = {"timeout": self.timeout, "scope": self.project_id} with metrics.timer("events.symbolicator.create_task", tags={"path": path}): return self._request(method="post", path=path, params=params, **kwargs) def symbolicate_stacktraces(self, stacktraces, modules, signal=None): json = { "sources": self.sources, "options": self.options, "stacktraces": stacktraces, "modules": modules, } if signal: json["signal"] = signal return self._create_task("symbolicate", json=json) def upload_minidump(self, minidump): return self._create_task( path="minidump", data={ "sources": json.dumps(self.sources), "options": json.dumps(self.options) }, files={"upload_file_minidump": minidump}, ) def upload_applecrashreport(self, report): return self._create_task( path="applecrashreport", data={ "sources": json.dumps(self.sources), "options": json.dumps(self.options) }, files={"apple_crash_report": report}, ) def query_task(self, task_id): task_url = f"requests/{task_id}" params = { "timeout": 0, # Only wait when creating, but not when querying tasks "scope": self.project_id, } with metrics.timer("events.symbolicator.query_task"): return self._request("get", task_url, params=params) def healthcheck(self): return self._request("get", "healthcheck")
class SymbolicatorSession(object): def __init__(self, url=None, sources=None, project_id=None, event_id=None, timeout=None): self.url = url self.project_id = project_id self.event_id = event_id self.sources = sources or [] self.timeout = timeout self.session = None self._query_params = {"timeout": timeout, "scope": project_id} def __enter__(self): self.open() return self def __exit__(self, *args): self.close() def open(self): if self.session is None: self.session = Session() def close(self): if self.session is not None: self.session.close() self.session = None def _ensure_open(self): if not self.session: raise RuntimeError("Session not opened") def _request(self, method, path, **kwargs): self._ensure_open() url = urljoin(self.url, path) # required for load balancing kwargs.setdefault("headers", {})["x-sentry-project-id"] = self.project_id kwargs.setdefault("headers", {})["x-sentry-event-id"] = self.event_id attempts = 0 wait = 0.5 while True: try: response = self.session.request(method, url, **kwargs) metrics.incr( "events.symbolicator.status_code", tags={ "status_code": response.status_code, "project_id": self.project_id }, ) if (method.lower() == "get" and path.startswith("requests/") and response.status_code == 404): # The symbolicator does not know this task. This is # expected to happen when we're currently deploying # symbolicator (which will clear all of its state). Re-send # the symbolication task. return None if response.status_code in (502, 503): raise ServiceUnavailable() if response.ok: json = response.json() else: json = { "status": "failed", "message": "internal server error" } return json except (IOError, RequestException): attempts += 1 # Any server error needs to be treated as a failure. We can # retry a couple of times, but ultimately need to bail out. # # This can happen for any network failure. if attempts > MAX_ATTEMPTS: logger.error("Failed to contact symbolicator", exc_info=True) raise time.sleep(wait) wait *= 2.0 def symbolicate_stacktraces(self, stacktraces, modules, signal=None): json = { "sources": self.sources, "stacktraces": stacktraces, "modules": modules } if signal: json["signal"] = signal return self._request("post", "symbolicate", params=self._query_params, json=json) def upload_minidump(self, minidump): return self._request( method="post", path="minidump", params=self._query_params, data={"sources": json.dumps(self.sources)}, files={"upload_file_minidump": minidump}, ) def upload_applecrashreport(self, report): return self._request( method="post", path="applecrashreport", params=self._query_params, data={"sources": json.dumps(self.sources)}, files={"apple_crash_report": report}, ) def query_task(self, task_id): task_url = "requests/%s" % (task_id, ) return self._request("get", task_url, params=self._query_params) def healthcheck(self): return self._request("get", "healthcheck")
def run_symbolicator(stacktraces, modules, project, arch, signal, request_id_cache_key): internal_url_prefix = options.get('system.internal-url-prefix') \ or options.get('system.url-prefix') assert internal_url_prefix sentry_source_url = '%s%s' % ( internal_url_prefix.rstrip('/'), reverse('sentry-api-0-dsym-files', kwargs={ 'organization_slug': project.organization.slug, 'project_slug': project.slug })) symbolicator_options = options.get('symbolicator.options') base_url = symbolicator_options['url'].rstrip('/') assert base_url project_id = six.text_type(project.id) request_id = default_cache.get(request_id_cache_key) sess = Session() attempts = 0 wait = 0.5 with sess: while 1: try: if request_id: rv = _poll_symbolication_task(sess=sess, base_url=base_url, request_id=request_id) else: rv = _create_symbolication_task( sess=sess, base_url=base_url, project_id=project_id, sentry_source_url=sentry_source_url, signal=signal, stacktraces=stacktraces, modules=modules) metrics.incr('events.symbolicator.status.%s' % rv.status_code, tags={'project_id': project_id}) if rv.status_code == 404 and request_id: default_cache.delete(request_id_cache_key) request_id = None continue elif rv.status_code == 503: raise RetrySymbolication(retry_after=10) rv.raise_for_status() json = rv.json() metrics.incr('events.symbolicator.response.%s' % json['status'], tags={'project_id': project_id}) if json['status'] == 'pending': default_cache.set(request_id_cache_key, json['request_id'], REQUEST_CACHE_TIMEOUT) raise RetrySymbolication(retry_after=json['retry_after']) elif json['status'] == 'completed': default_cache.delete(request_id_cache_key) return rv.json() else: logger.error("Unexpected status: %s", json['status']) default_cache.delete(request_id_cache_key) return except (IOError, RequestException): attempts += 1 if attempts > MAX_ATTEMPTS: logger.error('Failed to contact symbolicator', exc_info=True) default_cache.delete(request_id_cache_key) return time.sleep(wait) wait *= 2.0
class SymbolicatorSession(object): def __init__(self, url=None, sources=None, project_id=None, event_id=None, timeout=None): self.url = url self.project_id = project_id self.event_id = event_id self.sources = sources or [] self.timeout = timeout self.session = None self._query_params = {'timeout': timeout, 'scope': project_id} def __enter__(self): self.open() return self def __exit__(self, *args): self.close() def open(self): if self.session is None: self.session = Session() def close(self): if self.session is not None: self.session.close() self.session = None def _ensure_open(self): if not self.session: raise RuntimeError('Session not opened') def _request(self, method, path, **kwargs): self._ensure_open() url = urljoin(self.url, path) # required for load balancing kwargs.setdefault('headers', {})['x-sentry-project-id'] = self.project_id kwargs.setdefault('headers', {})['x-sentry-event-id'] = self.event_id attempts = 0 wait = 0.5 while True: try: response = self.session.request(method, url, **kwargs) metrics.incr('events.symbolicator.status_code', tags={ 'status_code': response.status_code, 'project_id': self.project_id, }) if (method.lower() == 'get' and path.startswith('requests/') and response.status_code == 404): # The symbolicator does not know this task. This is # expected to happen when we're currently deploying # symbolicator (which will clear all of its state). Re-send # the symbolication task. return None if response.status_code == 503: raise ServiceUnavailable() response.raise_for_status() json = response.json() return json except (IOError, RequestException): attempts += 1 # Any server error needs to be treated as a failure. We can # retry a couple of times, but ultimately need to bail out. # # This can happen for any network failure. if attempts > MAX_ATTEMPTS: logger.error('Failed to contact symbolicator', exc_info=True) raise time.sleep(wait) wait *= 2.0 def symbolicate_stacktraces(self, stacktraces, modules, signal=None): json = { 'sources': self.sources, 'stacktraces': stacktraces, 'modules': modules, } if signal: json['signal'] = signal return self._request('post', 'symbolicate', params=self._query_params, json=json) def upload_minidump(self, minidump): files = {'upload_file_minidump': minidump} data = { 'sources': json.dumps(self.sources), } return self._request('post', 'minidump', params=self._query_params, data=data, files=files) def query_task(self, task_id): task_url = 'requests/%s' % (task_id, ) return self._request('get', task_url, params=self._query_params) def healthcheck(self): return self._request('get', 'healthcheck')
def run_symbolicator(stacktraces, modules, project, arch, signal, request_id_cache_key): symbolicator_options = options.get('symbolicator.options') base_url = symbolicator_options['url'].rstrip('/') assert base_url project_id = six.text_type(project.id) request_id = default_cache.get(request_id_cache_key) sess = Session() # Will be set lazily when a symbolicator request is fired sources = None attempts = 0 wait = 0.5 with sess: while True: try: if request_id: rv = _poll_symbolication_task(sess=sess, base_url=base_url, request_id=request_id) else: if sources is None: sources = get_sources_for_project(project) rv = _create_symbolication_task(sess=sess, base_url=base_url, project_id=project_id, sources=sources, signal=signal, stacktraces=stacktraces, modules=modules) metrics.incr('events.symbolicator.status_code', tags={ 'status_code': rv.status_code, 'project_id': project_id, }) if rv.status_code == 404 and request_id: default_cache.delete(request_id_cache_key) request_id = None continue elif rv.status_code == 503: raise RetrySymbolication(retry_after=10) rv.raise_for_status() json = rv.json() metrics.incr('events.symbolicator.response', tags={ 'response': json['status'], 'project_id': project_id, }) if json['status'] == 'pending': default_cache.set(request_id_cache_key, json['request_id'], REQUEST_CACHE_TIMEOUT) raise RetrySymbolication(retry_after=json['retry_after']) elif json['status'] == 'completed': default_cache.delete(request_id_cache_key) return rv.json() else: logger.error("Unexpected status: %s", json['status']) default_cache.delete(request_id_cache_key) return except (IOError, RequestException): attempts += 1 if attempts > MAX_ATTEMPTS: logger.error('Failed to contact symbolicator', exc_info=True) default_cache.delete(request_id_cache_key) return time.sleep(wait) wait *= 2.0
class SymbolicatorSession: # used in x-sentry-worker-id http header # to keep it static for celery worker process keep it as class attribute _worker_id = None def __init__(self, url=None, sources=None, project_id=None, event_id=None, timeout=None, options=None): self.url = url self.project_id = project_id self.event_id = event_id self.sources = sources or [] self.options = options or None self.timeout = timeout self.session = None # Build some maps for use in ._process_response() self.reverse_source_aliases = reverse_aliases_map( settings.SENTRY_BUILTIN_SOURCES) self.source_names = { source["id"]: source.get("name", "unknown") for source in self.sources } # Add a name for the special "sentry:project" source. self.source_names[INTERNAL_SOURCE_NAME] = "Sentry" # Add names for aliased sources. for source in settings.SENTRY_BUILTIN_SOURCES.values(): if source.get("type") == "alias": self.source_names[source["id"]] = source.get("name", "unknown") # Remove sources that should be ignored. This leaves a few extra entries in the alias # maps and source names maps, but that's fine. The orphaned entries in the maps will just # never be used. self.sources = filter_ignored_sources(self.sources, self.reverse_source_aliases) def __enter__(self): self.open() return self def __exit__(self, *args): self.close() def open(self): if self.session is None: self.session = Session() def close(self): if self.session is not None: self.session.close() self.session = None def _ensure_open(self): if not self.session: raise RuntimeError("Session not opened") def _process_response(self, json): """Post-processes the JSON repsonse. This modifies the candidates list from Symbolicator responses to undo aliased sources, hide information about unknown sources and add names to sources rather then just have their IDs. """ for module in json.get("modules") or (): for candidate in module.get("candidates") or (): # Reverse internal source aliases from the response. source_id = candidate["source"] original_source_id = self.reverse_source_aliases.get(source_id) if original_source_id is not None: candidate["source"] = original_source_id source_id = original_source_id # Add a "source_name" field to save the UI a lookup. candidate["source_name"] = self.source_names.get( source_id, "unknown") redact_internal_sources(json) return json def _request(self, method, path, **kwargs): self._ensure_open() url = urljoin(self.url, path) # required for load balancing kwargs.setdefault("headers", {})["x-sentry-project-id"] = self.project_id kwargs.setdefault("headers", {})["x-sentry-event-id"] = self.event_id kwargs.setdefault("headers", {})["x-sentry-worker-id"] = self.get_worker_id() attempts = 0 wait = 0.5 while True: try: with metrics.timer("events.symbolicator.session.request", tags={"attempt": attempts}): response = self.session.request( method, url, timeout=settings.SYMBOLICATOR_POLL_TIMEOUT + 1, **kwargs) metrics.incr( "events.symbolicator.status_code", tags={ "status_code": response.status_code, "project_id": self.project_id }, ) if (method.lower() == "get" and path.startswith("requests/") and response.status_code == 404): # The symbolicator does not know this task. This is # expected to happen when we're currently deploying # symbolicator (which will clear all of its state). Re-send # the symbolication task. return None if response.status_code in (502, 503): raise ServiceUnavailable() if response.ok: json = response.json() else: json = { "status": "failed", "message": "internal server error" } return self._process_response(json) except (OSError, RequestException) as e: metrics.incr( "events.symbolicator.request_error", tags={ "exc": ".".join( [e.__class__.__module__, e.__class__.__name__]), "attempt": attempts, }, ) attempts += 1 # Any server error needs to be treated as a failure. We can # retry a couple of times, but ultimately need to bail out. # # This can happen for any network failure. if attempts > MAX_ATTEMPTS: logger.error("Failed to contact symbolicator", exc_info=True) raise time.sleep(wait) wait *= 2.0 def _create_task(self, path, **kwargs): params = {"timeout": self.timeout, "scope": self.project_id} with metrics.timer( "events.symbolicator.create_task", tags={ "path": path, "worker_id": self.get_worker_id() }, ): return self._request(method="post", path=path, params=params, **kwargs) def symbolicate_stacktraces(self, stacktraces, modules, signal=None): json = { "sources": self.sources, "options": self.options, "stacktraces": stacktraces, "modules": modules, } if signal: json["signal"] = signal return self._create_task("symbolicate", json=json) def upload_minidump(self, minidump): return self._create_task( path="minidump", data={ "sources": json.dumps(self.sources), "options": json.dumps(self.options) }, files={"upload_file_minidump": minidump}, ) def upload_applecrashreport(self, report): return self._create_task( path="applecrashreport", data={ "sources": json.dumps(self.sources), "options": json.dumps(self.options) }, files={"apple_crash_report": report}, ) def query_task(self, task_id): task_url = f"requests/{task_id}" params = { "timeout": 0, # Only wait when creating, but not when querying tasks "scope": self.project_id, } with metrics.timer("events.symbolicator.query_task", tags={"worker_id": self.get_worker_id()}): return self._request("get", task_url, params=params) def healthcheck(self): return self._request("get", "healthcheck") @classmethod def get_worker_id(cls): # as class attribute to keep it static for life of process if cls._worker_id is None: # %5000 to reduce cardinality of metrics tagging with worker id cls._worker_id = str(uuid.uuid4().int % 5000) return cls._worker_id