def _process(self, create_task, task_name):
task_id = default_cache.get(self.task_id_cache_key)
json_response = None
with self.sess:
try:
if task_id:
# Processing has already started and we need to poll
# symbolicator for an update. This in turn may put us back into
# the queue.
json_response = self.sess.query_task(task_id)
if json_response is None:
# This is a new task, so we compute all request parameters
# (potentially expensive if we need to pull minidumps), and then
# upload all information to symbolicator. It will likely not
# have a response ready immediately, so we start polling after
# some timeout.
json_response = create_task()
except ServiceUnavailable:
# 503 can indicate that symbolicator is restarting. Wait for a
# reboot, then try again. This overrides the default behavior of
# retrying after just a second.
#
# If there is no response attached, it's a connection error.
raise RetrySymbolication(
retry_after=settings.SYMBOLICATOR_MAX_RETRY_AFTER)
metrics.incr(
"events.symbolicator.response",
tags={
"response": json_response.get("status") or "null",
"task_name": task_name
},
)
# Symbolication is still in progress. Bail out and try again
# after some timeout. Symbolicator keeps the response for the
# first one to poll it.
if json_response["status"] == "pending":
default_cache.set(self.task_id_cache_key,
json_response["request_id"],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(
retry_after=json_response["retry_after"])
else:
# Once we arrive here, we are done processing. Clean up the
# task id from the cache.
default_cache.delete(self.task_id_cache_key)
metrics.timing("events.symbolicator.response.completed.size",
len(json.dumps(json_response)))
reverse_source_aliases(json_response)
redact_internal_sources(json_response)
return json_response
def _process(self, create_task):
task_id = default_cache.get(self.task_id_cache_key)
json = None
with self.sess:
try:
if task_id:
# Processing has already started and we need to poll
# symbolicator for an update. This in turn may put us back into
# the queue.
json = self.sess.query_task(task_id)
if json is None:
# This is a new task, so we compute all request parameters
# (potentially expensive if we need to pull minidumps), and then
# upload all information to symbolicator. It will likely not
# have a response ready immediately, so we start polling after
# some timeout.
json = create_task()
except ServiceUnavailable:
# 503 can indicate that symbolicator is restarting. Wait for a
# reboot, then try again. This overrides the default behavior of
# retrying after just a second.
#
# If there is no response attached, it's a connection error.
raise RetrySymbolication(retry_after=10)
metrics.incr('events.symbolicator.response', tags={
'response': json.get('status') or 'null',
'project_id': self.sess.project_id,
})
# Symbolication is still in progress. Bail out and try again
# after some timeout. Symbolicator keeps the response for the
# first one to poll it.
if json['status'] == 'pending':
default_cache.set(
self.task_id_cache_key,
json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
else:
# Once we arrive here, we are done processing. Clean up the
# task id from the cache.
default_cache.delete(self.task_id_cache_key)
return json
def run_symbolicator(stacktraces, modules, project, arch, signal,
request_id_cache_key):
internal_url_prefix = options.get('system.internal-url-prefix') \
or options.get('system.url-prefix')
assert internal_url_prefix
sentry_source_url = '%s%s' % (
internal_url_prefix.rstrip('/'),
reverse('sentry-api-0-dsym-files',
kwargs={
'organization_slug': project.organization.slug,
'project_slug': project.slug
}))
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
attempts = 0
wait = 0.5
with sess:
while 1:
try:
if request_id:
rv = _poll_symbolication_task(sess=sess,
base_url=base_url,
request_id=request_id)
else:
rv = _create_symbolication_task(
sess=sess,
base_url=base_url,
project_id=project_id,
sentry_source_url=sentry_source_url,
signal=signal,
stacktraces=stacktraces,
modules=modules)
metrics.incr('events.symbolicator.status.%s' % rv.status_code,
tags={'project_id': project_id})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response.%s' %
json['status'],
tags={'project_id': project_id})
if json['status'] == 'pending':
default_cache.set(request_id_cache_key, json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
elif json['status'] == 'completed':
default_cache.delete(request_id_cache_key)
return rv.json()
else:
logger.error("Unexpected status: %s", json['status'])
default_cache.delete(request_id_cache_key)
return
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator',
exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0
def run_symbolicator(stacktraces, modules, project, arch, signal,
request_id_cache_key):
symbolicator_options = options.get('symbolicator.options')
base_url = symbolicator_options['url'].rstrip('/')
assert base_url
project_id = six.text_type(project.id)
request_id = default_cache.get(request_id_cache_key)
sess = Session()
# Will be set lazily when a symbolicator request is fired
sources = None
attempts = 0
wait = 0.5
with sess:
while True:
try:
if request_id:
rv = _poll_symbolication_task(sess=sess,
base_url=base_url,
request_id=request_id)
else:
if sources is None:
sources = get_sources_for_project(project)
rv = _create_symbolication_task(sess=sess,
base_url=base_url,
project_id=project_id,
sources=sources,
signal=signal,
stacktraces=stacktraces,
modules=modules)
metrics.incr('events.symbolicator.status_code',
tags={
'status_code': rv.status_code,
'project_id': project_id,
})
if rv.status_code == 404 and request_id:
default_cache.delete(request_id_cache_key)
request_id = None
continue
elif rv.status_code == 503:
raise RetrySymbolication(retry_after=10)
rv.raise_for_status()
json = rv.json()
metrics.incr('events.symbolicator.response',
tags={
'response': json['status'],
'project_id': project_id,
})
if json['status'] == 'pending':
default_cache.set(request_id_cache_key, json['request_id'],
REQUEST_CACHE_TIMEOUT)
raise RetrySymbolication(retry_after=json['retry_after'])
elif json['status'] == 'completed':
default_cache.delete(request_id_cache_key)
return rv.json()
else:
logger.error("Unexpected status: %s", json['status'])
default_cache.delete(request_id_cache_key)
return
except (IOError, RequestException):
attempts += 1
if attempts > MAX_ATTEMPTS:
logger.error('Failed to contact symbolicator',
exc_info=True)
default_cache.delete(request_id_cache_key)
return
time.sleep(wait)
wait *= 2.0