Example #1
0
 def get_case_from_case_update(self, case_update, xform):
     """
     Gets or updates an existing case, based on a block of data in a
     submitted form.  Doesn't save anything. Returns a CaseUpdateMetadata object.
     """
     case = self.get(case_update.id)
     if case is None:
         if xform.metadata and xform.metadata.commcare_version:
             from distutils.version import LooseVersion
             commcare_version = xform.metadata.commcare_version
             message = "Case created without create block"
             send_to = None
             if commcare_version >= LooseVersion("2.39"):
                 send_to = "{}@{}.com".format('skelly', 'dimagi')
                 message += " in CC version >= 2.39"
             soft_assert(to=send_to)(case_update.creates_case(), message, {
                 'xform_id': xform.form_id,
                 'case_id': case_update.id,
                 'domain': xform.domain,
                 'version': str(commcare_version)
             })
         case = self.case_update_strategy.case_from_case_update(
             case_update, xform)
         self.set(case.case_id, case)
         return CaseUpdateMetadata(case,
                                   is_creation=True,
                                   previous_owner_id=None)
     else:
         previous_owner = case.owner_id
         self.case_update_strategy(case).update_from_case_update(
             case_update, xform, self.get_cached_forms())
         return CaseUpdateMetadata(case,
                                   is_creation=False,
                                   previous_owner_id=previous_owner)
Example #2
0
def get_case_ids_in_domain(domain, type=None):
    if type is None:
        type_keys = [[]]
    elif isinstance(type, (list, tuple)):
        soft_assert('skelly@{}'.format('dimagi.com'))(
            False, 'get_case_ids_in_domain called with typle / list arg for type'
        )
        type_keys = [[t] for t in type]
    elif isinstance(type, basestring):
        type_keys = [[type]]
    else:
        raise ValueError(
            "Argument type should be a string, tuple, or None: {!r}"
            .format(type)
        )
    return [
        res['id'] for type_key in type_keys
        for res in CommCareCase.get_db().view(
            'case_types_by_domain/view',
            startkey=[domain] + type_key,
            endkey=[domain] + type_key + [{}],
            reduce=False,
            include_docs=False,
        )
    ]
Example #3
0
def bulk_form_management_async(archive_or_restore, domain, couch_user, form_ids_or_query_string):
    # form_ids_or_query_string - can either be list of formids or a BulkFormMangementInterface query url
    def get_ids_from_url(query_string, domain, couch_user):
        from django.http import HttpRequest, QueryDict

        _request = HttpRequest()
        _request.couch_user = couch_user
        _request.user = couch_user.get_django_user()
        _request.domain = domain
        _request.couch_user.current_domain = domain

        _request.GET = QueryDict(query_string)
        dispatcher = EditDataInterfaceDispatcher()
        return dispatcher.dispatch(
            _request,
            render_as='form_ids',
            domain=domain,
            report_slug=BulkFormManagementInterface.slug,
            skip_permissions_check=True,
        )

    task = bulk_form_management_async
    mode = FormManagementMode(archive_or_restore, validate=True)

    if type(form_ids_or_query_string) == list:
        xform_ids = form_ids_or_query_string
    elif isinstance(form_ids_or_query_string, basestring):
        xform_ids = get_ids_from_url(form_ids_or_query_string, domain, couch_user)

    if not xform_ids:
        soft_assert(notify_admins=True, exponential_backoff=False)
        return {'messages': {'errors': [_('No Forms are supplied')]}}

    response = archive_or_restore_forms(domain, couch_user, xform_ids, mode, task)
    return response
Example #4
0
 def get_case_from_case_update(self, case_update, xform):
     """
     Gets or updates an existing case, based on a block of data in a
     submitted form.  Doesn't save anything. Returns a CaseUpdateMetadata object.
     """
     case = self.get(case_update.id)
     if case is None:
         if xform.metadata and xform.metadata.commcare_version:
             from distutils.version import LooseVersion
             commcare_version = xform.metadata.commcare_version
             message = "Case created without create block"
             send_to = None
             if commcare_version >= LooseVersion("2.44"):
                 send_to = "{}@{}.com".format('skelly', 'dimagi')
                 message += " in CC version >= 2.44"
             soft_assert(to=send_to)(
                 case_update.creates_case(),
                 message, {
                     'xform_id': xform.form_id,
                     'case_id': case_update.id,
                     'domain': xform.domain,
                     'version': str(commcare_version)
                 }
             )
         case = self.case_update_strategy.case_from_case_update(case_update, xform)
         self.set(case.case_id, case)
         return CaseUpdateMetadata(case, is_creation=True, previous_owner_id=None)
     else:
         previous_owner = case.owner_id
         self.case_update_strategy(case).update_from_case_update(case_update, xform, self.get_cached_forms())
         return CaseUpdateMetadata(case, is_creation=False, previous_owner_id=previous_owner)
Example #5
0
def get_case_ids_in_domain(domain, type=None):
    if type is None:
        type_keys = [[]]
    elif isinstance(type, (list, tuple)):
        soft_assert('skelly@{}'.format('dimagi.com'))(
            False,
            'get_case_ids_in_domain called with typle / list arg for type')
        type_keys = [[t] for t in type]
    elif isinstance(type, six.string_types):
        soft_assert_type_text(type)
        type_keys = [[type]]
    else:
        raise ValueError(
            "Argument type should be a string, tuple, or None: {!r}".format(
                type))
    return [
        res['id'] for type_key in type_keys
        for res in CommCareCase.get_db().view(
            'case_types_by_domain/view',
            startkey=[domain] + type_key,
            endkey=[domain] + type_key + [{}],
            reduce=False,
            include_docs=False,
        )
    ]
Example #6
0
def _handle_duplicate(new_doc):
    """
    Handle duplicate xforms and xform editing ('deprecation')

    existing doc *must* be validated as an XFormInstance in the right domain
    and *must* include inline attachments

    """
    interface = FormProcessorInterface(new_doc.domain)
    conflict_id = new_doc.form_id
    try:
        existing_doc = FormAccessors(
            new_doc.domain).get_with_attachments(conflict_id)
    except ResourceNotFound:
        # Original form processing failed but left behind a form doc with no
        # attachments. It's safe to delete this now since we're going to re-process
        # the form anyway.
        # https://sentry.io/dimagi/commcarehq/issues/261297321
        from couchforms.models import XFormInstance
        XFormInstance.get_db().delete_doc(conflict_id)
        return FormProcessingResult(new_doc)

    existing_md5 = existing_doc.xml_md5()
    new_md5 = new_doc.xml_md5()

    if existing_md5 != new_md5:
        if new_doc.xmlns != existing_doc.xmlns:
            # if the XMLNS has changed this probably isn't a form edit
            # it could be a UUID clash (yes we've had that before)
            # Assign a new ID to the form and process as normal + notify_admins
            xform = interface.assign_new_id(new_doc)
            soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'),
                        exponential_backoff=False)(
                            False, "Potential UUID clash", {
                                'incoming_form_id': conflict_id,
                                'existing_form_id': existing_doc.form_id,
                                'new_form_id': xform.form_id,
                                'incoming_xmlns': new_doc.xmlns,
                                'existing_xmlns': existing_doc.xmlns,
                                'domain': new_doc.domain,
                            })
            return FormProcessingResult(xform)
        else:
            # if the form contents are not the same:
            #  - "Deprecate" the old form by making a new document with the same contents
            #    but a different ID and a doc_type of XFormDeprecated
            #  - Save the new instance to the previous document to preserve the ID
            existing_doc, new_doc = apply_deprecation(existing_doc, new_doc,
                                                      interface)
            return FormProcessingResult(new_doc, existing_doc)
    else:
        # follow standard dupe handling, which simply saves a copy of the form
        # but a new doc_id, and a doc_type of XFormDuplicate
        duplicate = interface.deduplicate_xform(new_doc)
        return FormProcessingResult(duplicate, existing_doc)
Example #7
0
    def __init__(self, fn, vary_on, cache):
        self.fn = fn
        self.cache = cache
        self.prefix = '{}.{}'.format(
            fn.__name__[:40] + (fn.__name__[40:] and '..'),
            self._hash(inspect.getsource(fn), 8)
        )

        arg_names = inspect.getargspec(self.fn).args
        if not isfunction(vary_on):
            vary_on = [part.split('.') for part in vary_on]
            vary_on = [(part[0], tuple(part[1:])) for part in vary_on]
            for arg, attrs in vary_on:
                if arg not in arg_names:
                    raise ValueError(
                        'We cannot vary on "{}" because the function {} has '
                        'no such argument'.format(arg, self.fn.__name__)
                    )

        self.encoding_assert = soft_assert(
            to=['{}@{}'.format('skelly', 'dimagi.com')],
            fail_if_debug=False,
            skip_frames=5,
        )

        self.vary_on = vary_on
Example #8
0
    def __init__(self, fn, vary_on, cache):
        self.fn = fn
        self.cache = cache
        self.prefix = '{}.{}'.format(
            fn.__name__[:40] + (fn.__name__[40:] and '..'),
            self._hash(inspect.getsource(fn), 8))

        arg_names = inspect.getargspec(self.fn).args
        if not isfunction(vary_on):
            vary_on = [part.split('.') for part in vary_on]
            vary_on = [(part[0], tuple(part[1:])) for part in vary_on]
            for arg, attrs in vary_on:
                if arg not in arg_names:
                    raise ValueError(
                        'We cannot vary on "{}" because the function {} has '
                        'no such argument'.format(arg, self.fn.__name__))

        self.encoding_assert = soft_assert(
            notify_admins=True,
            fail_if_debug=False,
            skip_frames=5,
        )

        self.vary_on = vary_on
Example #9
0
import base64
import hashlib
import hmac
import logging
from functools import wraps

from django.conf import settings
from django.http import HttpResponse

from corehq.util.soft_assert.api import soft_assert

auth_logger = logging.getLogger("commcare_auth")
_soft_assert = soft_assert(notify_admins=True)


def convert_to_bytestring_if_unicode(shared_key):
    return shared_key.encode('utf-8') if isinstance(shared_key, str) else shared_key


def get_hmac_digest(shared_key, data):
    hm = hmac.new(convert_to_bytestring_if_unicode(shared_key), convert_to_bytestring_if_unicode(data), hashlib.sha256)
    digest = base64.b64encode(hm.digest())
    return digest.decode('utf-8')


def validate_request_hmac(setting_name, ignore_if_debug=False, audit_user='******'):
    """
    Decorator to validate request sender using a shared secret
    to compare the HMAC of the request body or query string with
    the value of the `X-MAC-DIGEST' header.
Example #10
0
def _handle_duplicate(new_doc):
    """
    Handle duplicate xforms and xform editing ('deprecation')

    existing doc *must* be validated as an XFormInstance in the right domain
    and *must* include inline attachments

    :returns: A two-tuple: `(<new form>, <duplicate form or None>)`
    The new form may have a different `form_id` than `new_doc.form_id`.
    """
    interface = FormProcessorInterface(new_doc.domain)
    conflict_id = new_doc.form_id
    try:
        existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id)
    except ResourceNotFound:
        # Original form processing failed but left behind a form doc with no
        # attachments. It's safe to delete this now since we're going to re-process
        # the form anyway.
        from couchforms.models import XFormInstance
        XFormInstance.get_db().delete_doc(conflict_id)
        return new_doc, None

    existing_md5 = existing_doc.xml_md5()
    new_md5 = new_doc.xml_md5()

    if existing_md5 != new_md5:
        _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False)
        if new_doc.xmlns != existing_doc.xmlns:
            # if the XMLNS has changed this probably isn't a form edit
            # it could be a UUID clash (yes we've had that before)
            # Assign a new ID to the form and process as normal + notify_admins
            xform = interface.assign_new_id(new_doc)
            _soft_assert(
                False, "Potential UUID clash", {
                    'incoming_form_id': conflict_id,
                    'existing_form_id': existing_doc.form_id,
                    'new_form_id': xform.form_id,
                    'incoming_xmlns': new_doc.xmlns,
                    'existing_xmlns': existing_doc.xmlns,
                    'domain': new_doc.domain,
                }
            )
            return xform, None
        else:
            # if the form contents are not the same:
            #  - "Deprecate" the old form by making a new document with the same contents
            #    but a different ID and a doc_type of XFormDeprecated
            #  - Save the new instance to the previous document to preserve the ID
            existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface)
            device_id = new_doc.metadata.deviceID
            # we expect edits from formplayer so exclude those
            _soft_assert(
                device_id == FORMPLAYER_DEVICE_ID, "Form edit", {
                    'form_id': new_doc.form_id,
                    'deprecated_form': existing_doc.form_id,
                    'domain': new_doc.domain,
                    'device_id': device_id,
                    'cc_version': new_doc.metadata.appVersion,
                }
            )
            return new_doc, existing_doc
    else:
        # follow standard dupe handling, which simply saves a copy of the form
        # but a new doc_id, and a doc_type of XFormDuplicate
        duplicate = interface.deduplicate_xform(new_doc)
        return duplicate, existing_doc
Example #11
0
from __future__ import absolute_import
from __future__ import unicode_literals
import base64
import hashlib
import hmac
from functools import wraps
import six

from django.conf import settings
from django.http import HttpResponse

from corehq.util.soft_assert.api import soft_assert

_soft_assert = soft_assert(notify_admins=True)


def convert_to_bytestring_if_unicode(shared_key):
    return shared_key.encode('utf-8') if isinstance(shared_key, six.text_type) else shared_key


def get_hmac_digest(shared_key, data):
    hm = hmac.new(convert_to_bytestring_if_unicode(shared_key), convert_to_bytestring_if_unicode(data), hashlib.sha256)
    digest = base64.b64encode(hm.digest())
    return digest


def validate_request_hmac(setting_name, ignore_if_debug=False):
    """
    Decorator to validate request sender using a shared secret
    to compare the HMAC of the request body or query string with
    the value of the `X-MAC-DIGEST' header.
Example #12
0
 def test_message_newlines(self):
     _soft_assert = soft_assert(notify_admins=True)
     _soft_assert(False, "don't\ncrash")
Example #13
0
import sys
import os
from couchdbkit import ResourceNotFound, ResourceConflict
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from http_parser.http import ParserError
from restkit import RequestError
from corehq.apps.domain.models import Domain
from corehq.apps.domainsync.management.commands.copy_utils import copy_postgres_data_for_docs
from corehq.util.dates import iso_string_to_date
from dimagi.utils.couch.database import get_db, iter_docs
from corehq.apps.domainsync.config import DocumentTransform, save
from couchdbkit.client import Database
from optparse import make_option
from corehq.util.soft_assert.api import soft_assert
_soft_assert = soft_assert('{}@{}'.format('tsheffels', 'dimagi.com'))

# doctypes we want to be careful not to copy, which must be explicitly
# specified with --include
from dimagi.utils.parsing import json_format_date

DEFAULT_EXCLUDE_TYPES = [
    'ReportNotification',
    'WeeklyNotification',
    'DailyNotification'
]

NUM_PROCESSES = 8


class Command(BaseCommand):
 def test_message_newlines(self):
     _soft_assert = soft_assert(notify_admins=True)
     _soft_assert(False, u"don't\ncrash")
Example #15
0
from abc import ABCMeta, abstractmethod, abstractproperty
import six
from casexml.apps.case.exceptions import IllegalCaseId
from corehq.util.soft_assert.api import soft_assert
from dimagi.utils.couch import release_lock
from corehq.form_processor.interfaces.processor import CaseUpdateMetadata, FormProcessorInterface

_soft_assert = soft_assert(to="{}@{}.com".format('skelly', 'dimagi'), notify_admins=True)


def _get_id_for_case(case):
    if isinstance(case, dict):
        return case['_id']
    return case.case_id


class AbstractCaseDbCache(six.with_metaclass(ABCMeta)):
    """
    A temp object we use to keep a cache of in-memory cases around
    so we can get the latest updates even if they haven't been saved
    to the database. Also provides some type checking safety.

    This class can be used as a re-entrant context manager:

    with case_db:
        case_db.get('case1')
        with case_db:
            case_db.get('case2')
    """

    @abstractproperty
Example #16
0
def _handle_duplicate(new_doc):
    """
    Handle duplicate xforms and xform editing ('deprecation')

    existing doc *must* be validated as an XFormInstance in the right domain
    and *must* include inline attachments

    :returns: A two-tuple: `(<new form>, <duplicate form or None>)`
    The new form may have a different `form_id` than `new_doc.form_id`.
    """
    interface = FormProcessorInterface(new_doc.domain)
    conflict_id = new_doc.form_id
    try:
        existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id)
    except ResourceNotFound:
        # Original form processing failed but left behind a form doc with no
        # attachments. It's safe to delete this now since we're going to re-process
        # the form anyway.
        from couchforms.models import XFormInstance
        XFormInstance.get_db().delete_doc(conflict_id)
        return new_doc, None

    try:
        existing_md5 = existing_doc.xml_md5()
    except MissingFormXml:
        existing_md5 = None
        if not existing_doc.is_error:
            existing_doc.problem = 'Missing form.xml'

    new_md5 = new_doc.xml_md5()

    if existing_md5 != new_md5:
        _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False)
        if new_doc.xmlns != existing_doc.xmlns:
            # if the XMLNS has changed this probably isn't a form edit
            # it could be a UUID clash (yes we've had that before)
            # Assign a new ID to the form and process as normal + notify_admins
            xform = interface.assign_new_id(new_doc)
            _soft_assert(
                False, "Potential UUID clash", {
                    'incoming_form_id': conflict_id,
                    'existing_form_id': existing_doc.form_id,
                    'new_form_id': xform.form_id,
                    'incoming_xmlns': new_doc.xmlns,
                    'existing_xmlns': existing_doc.xmlns,
                    'domain': new_doc.domain,
                }
            )
            return xform, None
        else:
            if existing_doc.is_error and not existing_doc.initial_processing_complete:
                # edge case from ICDS where a form errors and then future re-submissions of the same
                # form do not have the same MD5 hash due to a bug on mobile:
                # see https://dimagi-dev.atlassian.net/browse/ICDS-376

                # since we have a new form and the old one was not successfully processed
                # we can effectively ignore this form and process the new one as normal
                if not interface.use_sql_domain:
                    new_doc._rev, existing_doc._rev = existing_doc._rev, new_doc._rev
                interface.assign_new_id(existing_doc)
                existing_doc.save()
                return new_doc, None
            else:
                # if the form contents are not the same:
                #  - "Deprecate" the old form by making a new document with the same contents
                #    but a different ID and a doc_type of XFormDeprecated
                #  - Save the new instance to the previous document to preserve the ID
                NotAllowed.check(new_doc.domain)
                existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface)
                return new_doc, existing_doc
    else:
        # follow standard dupe handling, which simply saves a copy of the form
        # but a new doc_id, and a doc_type of XFormDuplicate
        duplicate = interface.deduplicate_xform(new_doc)
        return duplicate, existing_doc
Example #17
0
def _handle_duplicate(new_doc):
    """
    Handle duplicate xforms and xform editing ('deprecation')

    existing doc *must* be validated as an XFormInstance in the right domain
    and *must* include inline attachments

    :returns: A two-tuple: `(<new form>, <duplicate form or None>)`
    The new form may have a different `form_id` than `new_doc.form_id`.
    """
    interface = FormProcessorInterface(new_doc.domain)
    conflict_id = new_doc.form_id
    existing_doc = XFormInstance.objects.get_with_attachments(
        conflict_id, new_doc.domain)

    is_icds = settings.SERVER_ENVIRONMENT in settings.ICDS_ENVS
    try:
        if is_icds and new_doc.metadata.deviceID == existing_doc.metadata.deviceID:
            # ICDS does not use 'edit form' functionality via the web and form editing is not possible
            # on mobile devices so it's safe to assume this is a duplicate without checking md5 etc.
            duplicate = interface.deduplicate_xform(new_doc)
            return duplicate, existing_doc

        existing_md5 = existing_doc.xml_md5()
    except MissingFormXml:
        existing_md5 = new_md5 = None
        if not existing_doc.is_error:
            existing_doc.problem = 'Missing form.xml'
    else:
        # only do this if we need to do the comparison
        new_md5 = new_doc.xml_md5()

    if existing_md5 is None or existing_md5 != new_md5:

        def _deprecate_old_form():
            existing, new = apply_deprecation(existing_doc, new_doc, interface)
            return new, existing

        def _replace_old_form():
            interface.assign_new_id(existing_doc)
            existing_doc.orig_id = new_doc.form_id
            existing_doc.save()
            return new_doc, None

        _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'),
                                   exponential_backoff=False)
        if new_doc.xmlns != existing_doc.xmlns:
            # if the XMLNS has changed this probably isn't a form edit
            # it could be a UUID clash (yes we've had that before)
            # Assign a new ID to the form and process as normal + notify_admins
            xform = interface.assign_new_id(new_doc)
            _soft_assert(
                False, "Potential UUID clash", {
                    'incoming_form_id': conflict_id,
                    'existing_form_id': existing_doc.form_id,
                    'new_form_id': xform.form_id,
                    'incoming_xmlns': new_doc.xmlns,
                    'existing_xmlns': existing_doc.xmlns,
                    'domain': new_doc.domain,
                })
            return xform, None
        else:
            if existing_doc.is_error:
                # edge case from ICDS where a form errors and then future re-submissions of the same
                # form do not have the same MD5 hash due to a bug on mobile:
                # see https://dimagi-dev.atlassian.net/browse/ICDS-376
                if not existing_doc.initial_processing_complete:
                    # since we have a new form and the old one was not successfully processed
                    # we can effectively ignore this form and process the new one as normal
                    return _replace_old_form()
                elif not interface.form_has_case_transactions(
                        existing_doc.form_id):
                    # likely an error during saving
                    return _replace_old_form()
                else:
                    return _deprecate_old_form()
            else:
                # if the form contents are not the same:
                #  - "Deprecate" the old form by making a new document with the same contents
                #    but a different ID and a doc_type of XFormDeprecated
                #  - Save the new instance to the previous document to preserve the ID
                return _deprecate_old_form()
    else:
        # follow standard dupe handling, which simply saves a copy of the form
        # but a new doc_id, and a doc_type of XFormDuplicate
        duplicate = interface.deduplicate_xform(new_doc)
        return duplicate, existing_doc