def jserror(request):
agent = request.META.get('HTTP_USER_AGENT', None)
os = browser_name = browser_version = bot = DATADOG_UNKNOWN
if agent:
parsed_agent = httpagentparser.detect(agent)
bot = parsed_agent.get('bot', False)
if 'os' in parsed_agent:
os = parsed_agent['os'].get('name', DATADOG_UNKNOWN)
if 'browser' in parsed_agent:
browser_version = parsed_agent['browser'].get(
'version', DATADOG_UNKNOWN)
browser_name = parsed_agent['browser'].get('name', DATADOG_UNKNOWN)
log_counter(
JSERROR_COUNT, {
'os': os,
'browser_version': browser_version,
'browser_name': browser_name,
'url': sanitize_url(request.POST.get('page', None)),
'file': request.POST.get('filename'),
'bot': bot,
})
return HttpResponse('')
def get_context(self, record):
try:
request = record.request
except Exception:
request = None
request_repr = get_sanitized_request_repr(request)
tb_list = []
code = None
if record.exc_info:
etype, _value, tb = record.exc_info
value = clean_exception(_value)
tb_list = ['Traceback (most recent call first):\n']
formatted_exception = traceback.format_exception_only(etype, value)
tb_list.extend(formatted_exception)
extracted_tb = list(reversed(traceback.extract_tb(tb)))
code = self.get_code(extracted_tb)
tb_list.extend(traceback.format_list(extracted_tb))
stack_trace = '\n'.join(tb_list)
subject = '%s: %s' % (record.levelname,
formatted_exception[0].strip() if
formatted_exception else record.getMessage())
else:
stack_trace = 'No stack trace available'
subject = '%s: %s' % (record.levelname, record.getMessage())
context = defaultdict(lambda: '')
context.update({
'subject': self.format_subject(subject),
'message': record.getMessage(),
'details': getattr(record, 'details', None),
'tb_list': tb_list,
'request_repr': request_repr,
'stack_trace': stack_trace,
'code': code,
})
if request:
sanitized_url = sanitize_url(request.build_absolute_uri())
log_counter(
ERROR_COUNT, {
'url': sanitized_url,
'group': get_url_group(sanitized_url),
'domain': getattr(request, 'domain', DATADOG_UNKNOWN),
})
context.update({
'get':
request.GET.items(),
'post':
SafeExceptionReporterFilter().get_post_parameters(request),
'method':
request.method,
'username':
request.user.username
if getattr(request, 'user', None) else "",
'url':
request.build_absolute_uri(),
})
return context
def _fail(self, reason, response):
if self.repeater.allow_retries(response):
self.set_next_try()
self.failure_reason = reason
log_counter(REPEATER_ERROR_COUNT, {
'_id': self._id,
'reason': reason,
'target_url': self.url,
})
def __init__(self, user, synclog_id, **kwargs):
super(DateOpenedBugException, self).__init__(user, **kwargs)
details = {
"domain": user.domain,
"username": user.username,
"user_id": user.user_id,
"last_synclog_id": synclog_id
}
log_counter(DATE_OPENED_CASEBLOCK_ERROR_COUNT, details)
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
return response
def get_context(self, record):
try:
request = record.request
except Exception:
request = None
request_repr = get_sanitized_request_repr(request)
tb_list = []
code = None
if record.exc_info:
etype, _value, tb = record.exc_info
value = clean_exception(_value)
tb_list = ['Traceback (most recent call first):\n']
formatted_exception = traceback.format_exception_only(etype, value)
tb_list.extend(formatted_exception)
extracted_tb = list(reversed(traceback.extract_tb(tb)))
code = self.get_code(extracted_tb)
tb_list.extend(traceback.format_list(extracted_tb))
stack_trace = '\n'.join(tb_list)
subject = '%s: %s' % (record.levelname,
formatted_exception[0].strip() if formatted_exception else record.getMessage())
else:
stack_trace = 'No stack trace available'
subject = '%s: %s' % (
record.levelname,
record.getMessage()
)
context = defaultdict(lambda: '')
context.update({
'subject': self.format_subject(subject),
'message': record.getMessage(),
'details': getattr(record, 'details', None),
'tb_list': tb_list,
'request_repr': request_repr,
'stack_trace': stack_trace,
'code': code,
})
if request:
sanitized_url = sanitize_url(request.build_absolute_uri())
log_counter(ERROR_COUNT, {
'url': sanitized_url,
'group': get_url_group(sanitized_url),
'domain': getattr(request, 'domain', DATADOG_UNKNOWN),
})
context.update({
'get': request.GET,
'post': SafeExceptionReporterFilter().get_post_parameters(request),
'method': request.method,
'username': request.user.username if getattr(request, 'user', None) else "",
'url': request.build_absolute_uri(),
})
return context
def _fail(self, reason, response):
if self.repeater.allow_retries(
response) and self.overall_tries < self.max_possible_tries:
self.set_next_try()
else:
self.last_checked = datetime.utcnow()
self.cancel()
self.failure_reason = reason
log_counter(REPEATER_ERROR_COUNT, {
'_id': self._id,
'reason': reason,
'target_url': self.url,
})
def fire(self, max_tries=3, force_send=False):
try:
payload = self.get_payload()
except ResourceNotFound:
# this repeater is pointing at a missing document
# quarantine it and tell it to stop trying.
logging.exception(u'Repeater {} in domain {} references a missing or deleted document!'.format(
self._id, self.domain,
))
self.doc_type = self.doc_type + '-Failed'
self.save()
except IgnoreDocument:
# this repeater is pointing at a document with no payload
logging.info(u'Repeater {} in domain {} references a document with no payload'.format(
self._id, self.domain,
))
# Mark it succeeded so that we don't try again
self.update_success()
else:
headers = self.repeater.get_headers(self)
if self.try_now() or force_send:
# we don't use celery's version of retry because
# we want to override the success/fail each try
failure_reason = None
for i in range(max_tries):
try:
resp = simple_post_with_cached_timeout(
payload,
self.url,
headers=headers,
force_send=force_send,
timeout=POST_TIMEOUT,
)
if 200 <= resp.status_code < 300:
self.update_success()
break
else:
failure_reason = u'{}: {}'.format(resp.status_code, resp.reason)
except Exception, e:
failure_reason = unicode(e)
if not self.succeeded:
# mark it failed for later and give up
self.update_failure(failure_reason)
log_counter(REPEATER_ERROR_COUNT, {
'_id': self._id,
'reason': failure_reason,
'target_url': self.url,
})
def fire(self, max_tries=3, force_send=False):
try:
payload = self.get_payload()
except ResourceNotFound:
# this repeater is pointing at a missing document
# quarantine it and tell it to stop trying.
logging.exception(u'Repeater {} in domain {} references a missing or deleted document!'.format(
self._id, self.domain,
))
self.doc_type = self.doc_type + '-Failed'
self.save()
except IgnoreDocument:
# this repeater is pointing at a document with no payload
logging.info(u'Repeater {} in domain {} references a document with no payload'.format(
self._id, self.domain,
))
# Mark it succeeded so that we don't try again
self.update_success()
else:
headers = self.repeater.get_headers(self)
if self.try_now() or force_send:
# we don't use celery's version of retry because
# we want to override the success/fail each try
failure_reason = None
for i in range(max_tries):
try:
resp = simple_post_with_cached_timeout(
payload,
self.url,
headers=headers,
force_send=force_send,
timeout=POST_TIMEOUT,
)
if 200 <= resp.status_code < 300:
self.update_success()
break
else:
failure_reason = u'{}: {}'.format(resp.status_code, resp.reason)
except Exception, e:
failure_reason = unicode(e)
if not self.succeeded:
# mark it failed for later and give up
self.update_failure(failure_reason)
log_counter(REPEATER_ERROR_COUNT, {
'_id': self._id,
'reason': failure_reason,
'target_url': self.url,
})
def jserror(request):
agent = request.META.get('HTTP_USER_AGENT', None)
os = browser_name = browser_version = bot = DATADOG_UNKNOWN
if agent:
parsed_agent = httpagentparser.detect(agent)
bot = parsed_agent.get('bot', False)
if 'os' in parsed_agent:
os = parsed_agent['os'].get('name', DATADOG_UNKNOWN)
if 'browser' in parsed_agent:
browser_version = parsed_agent['browser'].get('version', DATADOG_UNKNOWN)
browser_name = parsed_agent['browser'].get('name', DATADOG_UNKNOWN)
log_counter(JSERROR_COUNT, {
'os': os,
'browser_version': browser_version,
'browser_name': browser_name,
'url': sanitize_url(request.POST.get('page', None)),
'file': request.POST.get('filename'),
'bot': bot,
})
return HttpResponse('')
def _process_form(request,
domain,
app_id,
user_id,
authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
return SubmissionPost.get_blacklisted_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(request, "Received a submission with POST.keys()",
details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
with TimingContext() as timer:
result = submission_post.run()
response = result.response
tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
u'domain:{}'.format(domain)
]
datadog_counter('commcare.xform_submissions.count',
tags=tags +
['status_code:{}'.format(response.status_code)])
if response.status_code == 400:
logging.error('Status code 400 for a form submission. '
'Response is: \n{0}\n')
elif response.status_code == 201:
datadog_gauge('commcare.xform_submissions.timings',
timer.duration,
tags=tags)
# normalize over number of items (form or case) saved
normalized_time = timer.duration / (1 + len(result.cases))
datadog_gauge('commcare.xform_submissions.normalized_timings',
normalized_time,
tags=tags)
datadog_counter('commcare.xform_submissions.case_count',
len(result.cases),
tags=tags)
datadog_counter('commcare.xform_submissions.ledger_count',
len(result.ledgers),
tags=tags)
return response
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
return SubmissionPost.submission_ignored_response()
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
details = {
"domain": domain,
"app_id": app_id,
"user_id": user_id,
"authenticated": authenticated,
"form_meta": meta,
}
log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
notify_exception(None, "Received a submission with POST.keys()", details)
return HttpResponseBadRequest(e.message)
app_id, build_id = get_app_and_build_ids(domain, app_id)
response = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
).get_response()
if response.status_code == 400:
db_response = get_db('couchlog').save_doc({
'request': unicode(request),
'response': unicode(response),
})
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
'See couchlog db for more info: {1}'.format(
unicode(response),
db_response['id'],
)
)
return response
