Example #1
0
def should_ignore_submission(request):
    """
    If IGNORE_ALL_DEMO_USER_SUBMISSIONS is True then ignore submission if from demo user.
    Else
    If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
    the submissions should be ignored
    """
    form_json = None
    if IGNORE_ALL_DEMO_USER_SUBMISSIONS:
        instance, _ = couchforms.get_instance_and_attachment(request)
        try:
            form_json = convert_xform_to_json(instance)
        except couchforms.XMLSyntaxError:
            # let the usual workflow handle response for invalid xml
            return False
        else:
            form_meta = form_json.get('meta')
            if form_meta and _submitted_by_demo_user(form_meta, request.domain):
                _notify_submission_if_applicable(request, form_meta)
                return True

    if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
        return False

    if form_json is None:
        instance, _ = couchforms.get_instance_and_attachment(request)
        form_json = convert_xform_to_json(instance)
    return False if from_demo_user(form_json) else True
Example #2
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    instance, attachments = couchforms.get_instance_and_attachment(request)
    response = couchforms.SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        db_response = get_db('couchlog').save_doc({
            'request': unicode(request),
            'response': unicode(response),
        })
        logging.error('Status code 400 for a form submission. '
                      'See couchlog db for more info: %s' % db_response['id'])
    return response
Example #3
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug as e:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        details = {
            "domain": domain,
            "app_id": app_id,
            "user_id": user_id,
            "authenticated": authenticated,
            "form_meta": meta,
        }
        log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
        notify_exception(None, "Received a submission with POST.keys()", details)
        return HttpResponseBadRequest(e.message)

    app_id, build_id = get_app_and_build_ids(domain, app_id)
    response = SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        db_response = get_db('couchlog').save_doc({
            'request': unicode(request),
            'response': unicode(response),
        })
        logging.error(
            'Status code 400 for a form submission. '
            'Response is: \n{0}\n'
            'See couchlog db for more info: {1}'.format(
                unicode(response),
                db_response['id'],
            )
        )
    return response
Example #4
0
def _noauth_post(request, domain, app_id=None):
    """
    This is explictly called for a submission that has secure submissions enabled, but is manually
    overriding the submit URL to not specify auth context. It appears to be used by demo mode.

    It mainly just checks that we are touching test data only in the right domain and submitting
    as demo_user.
    """
    instance, _ = couchforms.get_instance_and_attachment(request)
    form_json = convert_xform_to_json(instance)
    case_updates = get_case_updates(form_json)

    def form_ok(form_json):
        try:
            # require new-style meta/userID (reject Meta/chw_id)
            if form_json['meta']['userID'] == 'demo_user':
                return True
        except (KeyError, ValueError):
            pass
        if is_device_report(form_json):
            return True
        return False

    def case_block_ok(case_updates):
        """
        Check for all cases that we are submitting as demo_user and that the domain we
        are submitting against for any previously existing cases matches the submission
        domain.
        """
        allowed_ids = ('demo_user', 'demo_user_group_id', None)
        case_ids = set()
        for case_update in case_updates:
            case_ids.add(case_update.id)
            create_action = case_update.get_create_action()
            update_action = case_update.get_update_action()
            index_action = case_update.get_index_action()
            if create_action:
                if create_action.user_id not in allowed_ids:
                    return False
                if create_action.owner_id not in allowed_ids:
                    return False
            if update_action:
                if update_action.owner_id not in allowed_ids:
                    return False
            if index_action:
                for index in index_action.indices:
                    case_ids.add(index.referenced_id)

        # todo: consider whether we want to remove this call, and/or pass the result
        # through to the next function so we don't have to get the cases again later
        cases = CommCareCase.bulk_get_lite(list(case_ids))
        for case in cases:
            if case.domain != domain:
                return False
            if case.owner_id or case.user_id not in allowed_ids:
                return False
Example #5
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        return SubmissionPost.submission_ignored_response()

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        return SubmissionPost.get_blacklisted_response()

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug as e:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        details = {
            "domain": domain,
            "app_id": app_id,
            "user_id": user_id,
            "authenticated": authenticated,
            "form_meta": meta,
        }
        log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
        notify_exception(None, "Received a submission with POST.keys()", details)
        return HttpResponseBadRequest(e.message)

    app_id, build_id = get_app_and_build_ids(domain, app_id)
    response = SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        logging.error(
            'Status code 400 for a form submission. '
            'Response is: \n{0}\n'
        )
    return response
Example #6
0
def should_ignore_submission(request):
    """
    If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
    the submissions should be ignored
    """
    if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
        return False

    instance, _ = couchforms.get_instance_and_attachment(request)
    form_json = convert_xform_to_json(instance)

    return False if from_demo_user(form_json) else True
Example #7
0
def _noauth_post(request, domain, app_id=None):
    instance, _ = couchforms.get_instance_and_attachment(request)
    form_json = convert_xform_to_json(instance)
    case_updates = get_case_updates(form_json)

    def form_ok(form_json):
        try:
            # require new-style meta/userID (reject Meta/chw_id)
            if form_json['meta']['userID'] == 'demo_user':
                return True
        except (KeyError, ValueError):
            pass
        if is_device_report(form_json):
            return True
        return False

    def case_block_ok(case_updates):
        case_ids = set()
        for case_update in case_updates:
            case_ids.add(case_update.id)
            create_action = case_update.get_create_action()
            update_action = case_update.get_update_action()
            index_action = case_update.get_index_action()
            if create_action:
                if create_action.user_id not in ('demo_user', None):
                    return False
                if create_action.owner_id not in ('demo_user', None):
                    return False
            if update_action:
                if update_action.owner_id not in ('demo_user', None):
                    return False
            if index_action:
                for index in index_action.indices:
                    case_ids.add(index.referenced_id)
        cases = CommCareCase.bulk_get_lite(list(case_ids))
        for case in cases:
            if case.domain != domain:
                return False
            if case.owner_id or case.user_id != 'demo_user':
                return False
        return True

    if not (form_ok(form_json) and case_block_ok(case_updates)):
        return HttpResponseForbidden()

    return _process_form(
        request=request,
        domain=domain,
        app_id=app_id,
        user_id=None,
        authenticated=False,
        auth_cls=WaivedAuthContext,
    )
Example #8
0
def should_ignore_submission(request):
    """
    If submission request.GET has `submit_mode=demo` and submitting user is not demo_user,
    the submissions should be ignored
    """
    if not request.GET.get('submit_mode') == DEMO_SUBMIT_MODE:
        return False

    instance, _ = couchforms.get_instance_and_attachment(request)
    form_json = convert_xform_to_json(instance)

    return False if from_demo_user(form_json) else True
Example #9
0
def _noauth_post(request, domain, app_id=None):
    """
    This is explicitly called for a submission that has secure submissions enabled, but is manually
    overriding the submit URL to not specify auth context. It appears to be used by demo mode.

    It mainly just checks that we are touching test data only in the right domain and submitting
    as demo_user.
    """
    try:
        instance, _ = couchforms.get_instance_and_attachment(request)
    except BadSubmissionRequest as e:
        return HttpResponseBadRequest(e.message)

    form_json = convert_xform_to_json(instance)
    case_updates = get_case_updates(form_json)

    def form_ok(form_json):
        return (from_demo_user(form_json) or is_device_report(form_json))

    def case_block_ok(case_updates):
        """
        Check for all cases that we are submitting as demo_user and that the domain we
        are submitting against for any previously existing cases matches the submission
        domain.
        """
        allowed_ids = ('demo_user', 'demo_user_group_id', None)
        case_ids = set()
        for case_update in case_updates:
            case_ids.add(case_update.id)
            create_action = case_update.get_create_action()
            update_action = case_update.get_update_action()
            index_action = case_update.get_index_action()
            if create_action:
                if create_action.user_id not in allowed_ids:
                    return False
                if create_action.owner_id not in allowed_ids:
                    return False
            if update_action:
                if update_action.owner_id not in allowed_ids:
                    return False
            if index_action:
                for index in index_action.indices:
                    case_ids.add(index.referenced_id)

        # todo: consider whether we want to remove this call, and/or pass the result
        # through to the next function so we don't have to get the cases again later
        cases = CommCareCase.objects.get_cases(list(case_ids), domain)
        for case in cases:
            if case.domain != domain:
                return False
            if case.owner_id or case.user_id not in allowed_ids:
                return False
Example #10
0
def post(request):
    """
    XForms can get posted here.  They will be forwarded to couch.
    
    Just like play, if you specify a callback you get called, 
    otherwise you get a generic response.  Callbacks follow
    a different signature as play, only passing in the document
    (since we don't know what xform was being posted to)
    """
    instance, attachments = couchforms.get_instance_and_attachment(request)
    return SubmissionPost(
        instance=instance,
        attachments=attachments,
        path=couchforms.get_path(request),
    ).get_response()
Example #11
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):
    instance, attachments = couchforms.get_instance_and_attachment(request)
    app_id, build_id = get_app_and_build_ids(domain, app_id)
    response = couchforms.SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        db_response = get_db('couchlog').save_doc({
            'request':
            unicode(request),
            'response':
            unicode(response),
        })
        logging.error('Status code 400 for a form submission. '
                      'Response is: \n{0}\n'
                      'See couchlog db for more info: {1}'.format(
                          unicode(response),
                          db_response['id'],
                      ))
    return response
Example #12
0
def _noauth_post(request, domain, app_id=None):
    instance, _ = couchforms.get_instance_and_attachment(request)
    form_json = convert_xform_to_json(instance)
    case_updates = get_case_updates(form_json)

    def form_ok(form_json):
        try:
            # require new-style meta/userID (reject Meta/chw_id)
            if form_json['meta']['userID'] == 'demo_user':
                return True
        except (KeyError, ValueError):
            pass
        if is_device_report(form_json):
            return True
        return False

    def case_block_ok(case_updates):
        case_ids = set()
        for case_update in case_updates:
            case_ids.add(case_update.id)
            create_action = case_update.get_create_action()
            update_action = case_update.get_update_action()
            index_action = case_update.get_index_action()
            if create_action:
                if create_action.user_id not in ('demo_user', None):
                    return False
                if create_action.owner_id not in ('demo_user', None):
                    return False
            if update_action:
                if update_action.owner_id not in ('demo_user', None):
                    return False
            if index_action:
                for index in index_action.indices:
                    case_ids.add(index.referenced_id)
        cases = CommCareCase.bulk_get_lite(list(case_ids))
        for case in cases:
            if case.domain != domain:
                return False
            if case.owner_id or case.user_id != 'demo_user':
                return False
Example #13
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):
    metric_tags = [
        'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
        'domain:{}'.format(domain),
    ]
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        try:
            instance, attachments = couchforms.get_instance_and_attachment(
                request)
        except MultimediaBug as e:
            try:
                instance = request.FILES[MAGIC_PROPERTY].read()
                xform = convert_xform_to_json(instance)
                meta = xform.get("meta", {})
            except:
                meta = {}

            details = [
                "domain:{}".format(domain),
                "app_id:{}".format(app_id),
                "user_id:{}".format(user_id),
                "authenticated:{}".format(authenticated),
                "form_meta:{}".format(meta),
            ]
            datadog_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, tags=details)
            notify_exception(request, "Received a submission with POST.keys()",
                             details)
            response = HttpResponseBadRequest(six.text_type(e))
            _record_metrics(metric_tags, 'unknown', response)
            return response

        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
        )

        result = submission_post.run()

    response = result.response

    if response.status_code == 400:
        logging.error('Status code 400 for a form submission. '
                      'Response is: \n{0}\n')

    _record_metrics(metric_tags, result.submission_type, response, result,
                    timer)

    return response
Example #14
0
def _noauth_post(request, domain, app_id=None):
    """
    This is explictly called for a submission that has secure submissions enabled, but is manually
    overriding the submit URL to not specify auth context. It appears to be used by demo mode.

    It mainly just checks that we are touching test data only in the right domain and submitting
    as demo_user.
    """
    instance, _ = couchforms.get_instance_and_attachment(request)
    form_json = convert_xform_to_json(instance)
    case_updates = get_case_updates(form_json)

    def form_ok(form_json):
        try:
            # require new-style meta/userID (reject Meta/chw_id)
            if form_json['meta']['userID'] == 'demo_user':
                return True
        except (KeyError, ValueError):
            pass
        if is_device_report(form_json):
            return True
        return False

    def case_block_ok(case_updates):
        """
        Check for all cases that we are submitting as demo_user and that the domain we
        are submitting against for any previously existing cases matches the submission
        domain.
        """
        allowed_ids = ('demo_user', 'demo_user_group_id', None)
        case_ids = set()
        for case_update in case_updates:
            case_ids.add(case_update.id)
            create_action = case_update.get_create_action()
            update_action = case_update.get_update_action()
            index_action = case_update.get_index_action()
            if create_action:
                if create_action.user_id not in allowed_ids:
                    return False
                if create_action.owner_id not in allowed_ids:
                    return False
            if update_action:
                if update_action.owner_id not in allowed_ids:
                    return False
            if index_action:
                for index in index_action.indices:
                    case_ids.add(index.referenced_id)

        # todo: consider whether we want to remove this call, and/or pass the result
        # through to the next function so we don't have to get the cases again later
        cases = CommCareCase.bulk_get_lite(list(case_ids))
        for case in cases:
            if case.domain != domain:
                return False
            if case.owner_id or case.user_id not in allowed_ids:
                return False
        return True

    if not (form_ok(form_json) and case_block_ok(case_updates)):
        return HttpResponseForbidden()

    return _process_form(
        request=request,
        domain=domain,
        app_id=app_id,
        user_id=None,
        authenticated=False,
        auth_cls=WaivedAuthContext,
    )
Example #15
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):

    if rate_limit_submission(domain):
        return HttpTooManyRequests()

    metric_tags = {
        'backend': 'sql' if should_use_sql_backend(domain) else 'couch',
        'domain': domain
    }

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        metrics_counter('commcare.corrupt_multimedia_submissions',
                        tags={
                            'domain': domain,
                            'authenticated': authenticated
                        })
        return _submission_error(
            request,
            "Received a submission with POST.keys()",
            metric_tags,
            domain,
            app_id,
            user_id,
            authenticated,
            meta,
        )

    if isinstance(instance, BadRequest):
        response = HttpResponseBadRequest(instance.message)
        _record_metrics(metric_tags, 'known_failures', response)
        return response

    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
            force_logs=request.GET.get('force_logs', 'false') == 'true',
        )

        try:
            result = submission_post.run()
        except XFormLockError as err:
            metrics_counter('commcare.xformlocked.count',
                            tags={
                                'domain': domain,
                                'authenticated': authenticated
                            })
            return _submission_error(
                request,
                "XFormLockError: %s" % err,
                metric_tags,
                domain,
                app_id,
                user_id,
                authenticated,
                status=423,
                notify=False,
            )

    response = result.response
    _record_metrics(metric_tags, result.submission_type, result.response,
                    timer, result.xform)

    return response
Example #16
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        return SubmissionPost.submission_ignored_response()

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug as e:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        details = {
            "domain": domain,
            "app_id": app_id,
            "user_id": user_id,
            "authenticated": authenticated,
            "form_meta": meta,
        }
        log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
        notify_exception(None, "Received a submission with POST.keys()", details)
        return HttpResponseBadRequest(e.message)

    app_id, build_id = get_app_and_build_ids(domain, app_id)
    response = SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    ).get_response()
    if response.status_code == 400:
        db_response = get_db('couchlog').save_doc({
            'request': unicode(request),
            'response': unicode(response),
        })
        logging.error(
            'Status code 400 for a form submission. '
            'Response is: \n{0}\n'
            'See couchlog db for more info: {1}'.format(
                unicode(response),
                db_response['id'],
            )
        )
    return response
Example #17
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        return SubmissionPost.submission_ignored_response()

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        return SubmissionPost.get_blacklisted_response()

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug as e:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except:
            meta = {}

        details = {
            "domain": domain,
            "app_id": app_id,
            "user_id": user_id,
            "authenticated": authenticated,
            "form_meta": meta,
        }
        log_counter(MULTIMEDIA_SUBMISSION_ERROR_COUNT, details)
        notify_exception(request, "Received a submission with POST.keys()",
                         details)
        return HttpResponseBadRequest(e.message)

    app_id, build_id = get_app_and_build_ids(domain, app_id)
    submission_post = SubmissionPost(
        instance=instance,
        attachments=attachments,
        domain=domain,
        app_id=app_id,
        build_id=build_id,
        auth_context=auth_cls(
            domain=domain,
            user_id=user_id,
            authenticated=authenticated,
        ),
        location=couchforms.get_location(request),
        received_on=couchforms.get_received_on(request),
        date_header=couchforms.get_date_header(request),
        path=couchforms.get_path(request),
        submit_ip=couchforms.get_submit_ip(request),
        last_sync_token=couchforms.get_last_sync_token(request),
        openrosa_headers=couchforms.get_openrosa_headers(request),
    )
    with TimingContext() as timer:
        result = submission_post.run()

    response = result.response

    tags = [
        'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
        u'domain:{}'.format(domain)
    ]
    datadog_counter('commcare.xform_submissions.count',
                    tags=tags +
                    ['status_code:{}'.format(response.status_code)])

    if response.status_code == 400:
        logging.error('Status code 400 for a form submission. '
                      'Response is: \n{0}\n')
    elif response.status_code == 201:

        datadog_gauge('commcare.xform_submissions.timings',
                      timer.duration,
                      tags=tags)
        # normalize over number of items (form or case) saved
        normalized_time = timer.duration / (1 + len(result.cases))
        datadog_gauge('commcare.xform_submissions.normalized_timings',
                      normalized_time,
                      tags=tags)
        datadog_counter('commcare.xform_submissions.case_count',
                        len(result.cases),
                        tags=tags)
        datadog_counter('commcare.xform_submissions.ledger_count',
                        len(result.ledgers),
                        tags=tags)

    return response
Example #18
0
def _process_form(request, domain, app_id, user_id, authenticated,
                  auth_cls=AuthContext):
    metric_tags = [
        'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
        'domain:{}'.format(domain),
    ]
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        try:
            instance, attachments = couchforms.get_instance_and_attachment(request)
        except MultimediaBug as e:
            try:
                instance = request.FILES[MAGIC_PROPERTY].read()
                xform = convert_xform_to_json(instance)
                meta = xform.get("meta", {})
            except:
                meta = {}

            return _submission_error(
                request, "Received a submission with POST.keys()",
                MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags,
                domain, app_id, user_id, authenticated, meta,
            )

        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
        )

        try:
            result = submission_post.run()
        except XFormLockError as err:
            return _submission_error(
                request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT,
                metric_tags, domain, app_id, user_id, authenticated, status=423,
                notify=False,
            )

    response = result.response

    if response.status_code == 400:
        logging.error(
            'Status code 400 for a form submission. '
            'Response is: \n{0}\n'
        )

    _record_metrics(metric_tags, result.submission_type, response, result, timer)

    return response
Example #19
0
def _noauth_post(request, domain, app_id=None):
    """
    This is explictly called for a submission that has secure submissions enabled, but is manually
    overriding the submit URL to not specify auth context. It appears to be used by demo mode.

    It mainly just checks that we are touching test data only in the right domain and submitting
    as demo_user.
    """
    instance, _ = couchforms.get_instance_and_attachment(request)
    form_json = convert_xform_to_json(instance)
    case_updates = get_case_updates(form_json)

    def form_ok(form_json):
        return (from_demo_user(form_json) or is_device_report(form_json))

    def case_block_ok(case_updates):
        """
        Check for all cases that we are submitting as demo_user and that the domain we
        are submitting against for any previously existing cases matches the submission
        domain.
        """
        allowed_ids = ('demo_user', 'demo_user_group_id', None)
        case_ids = set()
        for case_update in case_updates:
            case_ids.add(case_update.id)
            create_action = case_update.get_create_action()
            update_action = case_update.get_update_action()
            index_action = case_update.get_index_action()
            if create_action:
                if create_action.user_id not in allowed_ids:
                    return False
                if create_action.owner_id not in allowed_ids:
                    return False
            if update_action:
                if update_action.owner_id not in allowed_ids:
                    return False
            if index_action:
                for index in index_action.indices:
                    case_ids.add(index.referenced_id)

        # todo: consider whether we want to remove this call, and/or pass the result
        # through to the next function so we don't have to get the cases again later
        cases = CaseAccessors(domain).get_cases(list(case_ids))
        for case in cases:
            if case.domain != domain:
                return False
            if case.owner_id or case.user_id not in allowed_ids:
                return False
        return True

    if not (form_ok(form_json) and case_block_ok(case_updates)):
        if request.GET.get('submit_mode') != DEMO_SUBMIT_MODE:
            # invalid submissions under demo mode submission can be processed
            return HttpResponseForbidden()

    return _process_form(
        request=request,
        domain=domain,
        app_id=app_id,
        user_id=None,
        authenticated=False,
        auth_cls=WaivedAuthContext,
    )
Example #20
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):

    if rate_limit_submission(domain):
        return HttpTooManyRequests()

    metric_tags = {'backend': 'sql', 'domain': domain}

    try:
        instance, attachments = couchforms.get_instance_and_attachment(request)
    except MultimediaBug:
        try:
            instance = request.FILES[MAGIC_PROPERTY].read()
            xform = convert_xform_to_json(instance)
            meta = xform.get("meta", {})
        except Exception:
            meta = {}

        metrics_counter('commcare.corrupt_multimedia_submissions',
                        tags={
                            'domain': domain,
                            'authenticated': authenticated
                        })
        return _submission_error(
            request,
            "Received a submission with POST.keys()",
            metric_tags,
            domain,
            app_id,
            user_id,
            authenticated,
            meta,
        )
    # the order of these exceptions is relevant
    except UnprocessableFormSubmission as e:
        return openrosa_response.OpenRosaResponse(
            message=e.message,
            nature=openrosa_response.ResponseNature.PROCESSING_FAILURE,
            status=e.status_code,
        ).response()
    except BadSubmissionRequest as e:
        response = HttpResponse(e.message, status=e.status_code)
        _record_metrics(metric_tags, 'known_failures', response)
        return response

    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
            force_logs=request.GET.get('force_logs', 'false') == 'true',
            timing_context=timer)

        try:
            result = submission_post.run()
        except XFormLockError as err:
            logging.warning('Unable to get lock for form %s', err)
            metrics_counter('commcare.xformlocked.count',
                            tags={
                                'domain': domain,
                                'authenticated': authenticated
                            })
            return _submission_error(
                request,
                "XFormLockError: %s" % err,
                metric_tags,
                domain,
                app_id,
                user_id,
                authenticated,
                status=423,
                notify=False,
            )

    response = result.response
    response.request_timer = timer  # logged as Sentry breadcrumbs in LogLongRequestMiddleware

    _record_metrics(metric_tags, result.submission_type, result.response,
                    timer, result.xform)

    return response
Example #21
0
def _process_form(request,
                  domain,
                  app_id,
                  user_id,
                  authenticated,
                  auth_cls=AuthContext):

    rate_limit_submission_by_delaying(domain, max_wait=15)

    metric_tags = [
        'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
        'domain:{}'.format(domain),
    ]
    if should_ignore_submission(request):
        # silently ignore submission if it meets ignore-criteria
        response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
        _record_metrics(metric_tags, 'ignored', response)
        return response

    if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
        response = openrosa_response.BLACKLISTED_RESPONSE
        _record_metrics(metric_tags, 'blacklisted', response)
        return response

    with TimingContext() as timer:
        try:
            instance, attachments = couchforms.get_instance_and_attachment(
                request)
        except MultimediaBug as e:
            try:
                instance = request.FILES[MAGIC_PROPERTY].read()
                xform = convert_xform_to_json(instance)
                meta = xform.get("meta", {})
            except:
                meta = {}

            return _submission_error(
                request,
                "Received a submission with POST.keys()",
                MULTIMEDIA_SUBMISSION_ERROR_COUNT,
                metric_tags,
                domain,
                app_id,
                user_id,
                authenticated,
                meta,
            )

        app_id, build_id = get_app_and_build_ids(domain, app_id)
        submission_post = SubmissionPost(
            instance=instance,
            attachments=attachments,
            domain=domain,
            app_id=app_id,
            build_id=build_id,
            auth_context=auth_cls(
                domain=domain,
                user_id=user_id,
                authenticated=authenticated,
            ),
            location=couchforms.get_location(request),
            received_on=couchforms.get_received_on(request),
            date_header=couchforms.get_date_header(request),
            path=couchforms.get_path(request),
            submit_ip=couchforms.get_submit_ip(request),
            last_sync_token=couchforms.get_last_sync_token(request),
            openrosa_headers=couchforms.get_openrosa_headers(request),
            force_logs=bool(request.GET.get('force_logs', False)),
        )

        try:
            result = submission_post.run()
        except XFormLockError as err:
            return _submission_error(
                request,
                "XFormLockError: %s" % err,
                XFORM_LOCKED_COUNT,
                metric_tags,
                domain,
                app_id,
                user_id,
                authenticated,
                status=423,
                notify=False,
            )

    response = result.response
    _record_metrics(metric_tags, result.submission_type, result.response,
                    timer, result.xform)

    return response