Example #1
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    instance, attachments = couchforms.get_instance_and_attachment(request)
    response = couchforms.SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        db_response = get_db('couchlog').save_doc({
            'request': unicode(request),
            'response': unicode(response),
        })
        logging.error('Status code 400 for a form submission. '
                      'See couchlog db for more info: %s' % db_response['id'])
    return response
Example #2
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug as e:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        details = {
            "domain": domain,
            "app_id": app_id,
            "user_id": user_id,
            "authenticated": authenticated,
            "form_meta": meta,
        }
        log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
        notify_exception(None, "Received a submission with POST.keys()", details)
        return HttpResponseBadRequest(e.message)

    app_id, build_id = get_app_and_build_ids(domain, app_id)
    response = SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        db_response = get_db('couchlog').save_doc({
            'request': unicode(request),
            'response': unicode(response),
        })
        logging.error(
            'Status code 400 for a form submission. '
            'Response is: \n{0}\n'
            'See couchlog db for more info: {1}'.format(
                unicode(response),
                db_response['id'],
            )
        )
    return response
Example #3
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        return SubmissionPost.submission_ignored_response()

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        return SubmissionPost.get_blacklisted_response()

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug as e:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        details = {
            "domain": domain,
            "app_id": app_id,
            "user_id": user_id,
            "authenticated": authenticated,
            "form_meta": meta,
        }
        log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
        notify_exception(None, "Received a submission with POST.keys()", details)
        return HttpResponseBadRequest(e.message)

    app_id, build_id = get_app_and_build_ids(domain, app_id)
    response = SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        logging.error(
            'Status code 400 for a form submission. '
            'Response is: \n{0}\n'
        )
    return response
Example #4
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):
    instance, attachments = couchforms.get_instance_and_attachment(request)
    app_id, build_id = get_app_and_build_ids(domain, app_id)
    response = couchforms.SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        db_response = get_db('couchlog').save_doc({
            'request':
            unicode(request),
            'response':
            unicode(response),
        })
        logging.error('Status code 400 for a form submission. '
                      'Response is: \n{0}\n'
                      'See couchlog db for more info: {1}'.format(
                          unicode(response),
                          db_response['id'],
                      ))
    return response
Example #5
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):

    if rate_limit_submission(domain):
        return HttpTooManyRequests()

    metric_tags = {'backend': 'sql', 'domain': domain}

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except Exception:
            meta = {}

        metrics_counter('commcare.corrupt_multimedia_submissions',
                        tags={
                            'domain': domain,
                            'authenticated': authenticated
                        })
        return _submission_error(
            request,
            "Received a submission with POST.keys()",
            metric_tags,
            domain,
            app_id,
            user_id,
            authenticated,
            meta,
        )
    # the order of these exceptions is relevant
    except UnprocessableFormSubmission as e:
        return openrosa_response.OpenRosaResponse(
            message=e.message,
            nature=openrosa_response.ResponseNature.PROCESSING_FAILURE,
            status=e.status_code,
        ).response()
    except BadSubmissionRequest as e:
        response = HttpResponse(e.message, status=e.status_code)
        _record_metrics(metric_tags, 'known_failures', response)
        return response

    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
            force_logs=request.GET.get('force_logs', 'false') == 'true',
            timing_context=timer)

        try:
            result = submission_post.run()
        except XFormLockError as err:
            logging.warning('Unable to get lock for form %s', err)
            metrics_counter('commcare.xformlocked.count',
                            tags={
                                'domain': domain,
                                'authenticated': authenticated
                            })
            return _submission_error(
                request,
                "XFormLockError: %s" % err,
                metric_tags,
                domain,
                app_id,
                user_id,
                authenticated,
                status=423,
                notify=False,
            )

    response = result.response
    response.request_timer = timer  # logged as Sentry breadcrumbs in LogLongRequestMiddleware

    _record_metrics(metric_tags, result.submission_type, result.response,
                    timer, result.xform)

    return response
Example #6
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):
    metric_tags = [
        'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
        'domain:{}'.format(domain),
    ]
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        try:
            instance, attachments = couchforms.get_instance_and_attachment(
                request)
        except MultimediaBug as e:
            try:
                instance = request.FILES[MAGIC_PROPERTY].read()
                xform = convert_xform_to_json(instance)
                meta = xform.get("meta", {})
            except:
                meta = {}

            details = [
                "domain:{}".format(domain),
                "app_id:{}".format(app_id),
                "user_id:{}".format(user_id),
                "authenticated:{}".format(authenticated),
                "form_meta:{}".format(meta),
            ]
            datadog_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, tags=details)
            notify_exception(request, "Received a submission with POST.keys()",
                             details)
            response = HttpResponseBadRequest(six.text_type(e))
            _record_metrics(metric_tags, 'unknown', response)
            return response

        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
        )

        result = submission_post.run()

    response = result.response

    if response.status_code == 400:
        logging.error('Status code 400 for a form submission. '
                      'Response is: \n{0}\n')

    _record_metrics(metric_tags, result.submission_type, response, result,
                    timer)

    return response
Example #7
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):

    if rate_limit_submission(domain):
        return HttpTooManyRequests()

    metric_tags = {
        'backend': 'sql' if should_use_sql_backend(domain) else 'couch',
        'domain': domain
    }

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        metrics_counter('commcare.corrupt_multimedia_submissions',
                        tags={
                            'domain': domain,
                            'authenticated': authenticated
                        })
        return _submission_error(
            request,
            "Received a submission with POST.keys()",
            metric_tags,
            domain,
            app_id,
            user_id,
            authenticated,
            meta,
        )

    if isinstance(instance, BadRequest):
        response = HttpResponseBadRequest(instance.message)
        _record_metrics(metric_tags, 'known_failures', response)
        return response

    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
            force_logs=request.GET.get('force_logs', 'false') == 'true',
        )

        try:
            result = submission_post.run()
        except XFormLockError as err:
            metrics_counter('commcare.xformlocked.count',
                            tags={
                                'domain': domain,
                                'authenticated': authenticated
                            })
            return _submission_error(
                request,
                "XFormLockError: %s" % err,
                metric_tags,
                domain,
                app_id,
                user_id,
                authenticated,
                status=423,
                notify=False,
            )

    response = result.response
    _record_metrics(metric_tags, result.submission_type, result.response,
                    timer, result.xform)

    return response
Example #8
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        return SubmissionPost.submission_ignored_response()

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        return SubmissionPost.get_blacklisted_response()

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug as e:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        details = {
            "domain": domain,
            "app_id": app_id,
            "user_id": user_id,
            "authenticated": authenticated,
            "form_meta": meta,
        }
        log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
        notify_exception(request, "Received a submission with POST.keys()",
                         details)
        return HttpResponseBadRequest(e.message)

    app_id, build_id = get_app_and_build_ids(domain, app_id)
    submission_post = SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    )
    with TimingContext() as timer:
        result = submission_post.run()

    response = result.response

    tags = [
        'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
        u'domain:{}'.format(domain)
    ]
    datadog_counter('commcare.xform_submissions.count',
                    tags=tags +
                    ['status_code:{}'.format(response.status_code)])

    if response.status_code == 400:
        logging.error('Status code 400 for a form submission. '
                      'Response is: \n{0}\n')
    elif response.status_code == 201:

        datadog_gauge('commcare.xform_submissions.timings',
                      timer.duration,
                      tags=tags)
        # normalize over number of items (form or case) saved
        normalized_time = timer.duration / (1 + len(result.cases))
        datadog_gauge('commcare.xform_submissions.normalized_timings',
                      normalized_time,
                      tags=tags)
        datadog_counter('commcare.xform_submissions.case_count',
                        len(result.cases),
                        tags=tags)
        datadog_counter('commcare.xform_submissions.ledger_count',
                        len(result.ledgers),
                        tags=tags)

    return response
Example #9
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    metric_tags = [
        'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
        'domain:{}'.format(domain),
    ]
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        try:
            instance, attachments = couchforms.get_instance_and_attachment(request)
        except MultimediaBug as e:
            try:
                instance = request.FILES[MAGIC_PROPERTY].read()
                xform = convert_xform_to_json(instance)
                meta = xform.get("meta", {})
            except:
                meta = {}

            return _submission_error(
                request, "Received a submission with POST.keys()",
                MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags,
                domain, app_id, user_id, authenticated, meta,
            )

        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
        )

        try:
            result = submission_post.run()
        except XFormLockError as err:
            return _submission_error(
                request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT,
                metric_tags, domain, app_id, user_id, authenticated, status=423,
                notify=False,
            )

    response = result.response

    if response.status_code == 400:
        logging.error(
            'Status code 400 for a form submission. '
            'Response is: \n{0}\n'
        )

    _record_metrics(metric_tags, result.submission_type, response, result, timer)

    return response
Example #10
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        return SubmissionPost.submission_ignored_response()

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug as e:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        details = {
            "domain": domain,
            "app_id": app_id,
            "user_id": user_id,
            "authenticated": authenticated,
            "form_meta": meta,
        }
        log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
        notify_exception(None, "Received a submission with POST.keys()", details)
        return HttpResponseBadRequest(e.message)

    app_id, build_id = get_app_and_build_ids(domain, app_id)
    response = SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        db_response = get_db('couchlog').save_doc({
            'request': unicode(request),
            'response': unicode(response),
        })
        logging.error(
            'Status code 400 for a form submission. '
            'Response is: \n{0}\n'
            'See couchlog db for more info: {1}'.format(
                unicode(response),
                db_response['id'],
            )
        )
    return response
Example #11
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):

    rate_limit_submission_by_delaying(domain, max_wait=15)

    metric_tags = [
        'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
        'domain:{}'.format(domain),
    ]
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        try:
            instance, attachments = couchforms.get_instance_and_attachment(
                request)
        except MultimediaBug as e:
            try:
                instance = request.FILES[MAGIC_PROPERTY].read()
                xform = convert_xform_to_json(instance)
                meta = xform.get("meta", {})
            except:
                meta = {}

            return _submission_error(
                request,
                "Received a submission with POST.keys()",
                MULTIMEDIA_SUBMISSION_ERROR_COUNT,
                metric_tags,
                domain,
                app_id,
                user_id,
                authenticated,
                meta,
            )

        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
            force_logs=bool(request.GET.get('force_logs', False)),
        )

        try:
            result = submission_post.run()
        except XFormLockError as err:
            return _submission_error(
                request,
                "XFormLockError: %s" % err,
                XFORM_LOCKED_COUNT,
                metric_tags,
                domain,
                app_id,
                user_id,
                authenticated,
                status=423,
                notify=False,
            )

    response = result.response
    _record_metrics(metric_tags, result.submission_type, result.response,
                    timer, result.xform)

    return response