def get_case_from_case_update(self, case_update, xform): """ Gets or updates an existing case, based on a block of data in a submitted form. Doesn't save anything. Returns a CaseUpdateMetadata object. """ case = self.get(case_update.id) if case is None: if xform.metadata and xform.metadata.commcare_version: from distutils.version import LooseVersion commcare_version = xform.metadata.commcare_version message = "Case created without create block" send_to = None if commcare_version >= LooseVersion("2.39"): send_to = "{}@{}.com".format('skelly', 'dimagi') message += " in CC version >= 2.39" soft_assert(to=send_to)(case_update.creates_case(), message, { 'xform_id': xform.form_id, 'case_id': case_update.id, 'domain': xform.domain, 'version': str(commcare_version) }) case = self.case_update_strategy.case_from_case_update( case_update, xform) self.set(case.case_id, case) return CaseUpdateMetadata(case, is_creation=True, previous_owner_id=None) else: previous_owner = case.owner_id self.case_update_strategy(case).update_from_case_update( case_update, xform, self.get_cached_forms()) return CaseUpdateMetadata(case, is_creation=False, previous_owner_id=previous_owner)
def get_case_ids_in_domain(domain, type=None): if type is None: type_keys = [[]] elif isinstance(type, (list, tuple)): soft_assert('skelly@{}'.format('dimagi.com'))( False, 'get_case_ids_in_domain called with typle / list arg for type' ) type_keys = [[t] for t in type] elif isinstance(type, basestring): type_keys = [[type]] else: raise ValueError( "Argument type should be a string, tuple, or None: {!r}" .format(type) ) return [ res['id'] for type_key in type_keys for res in CommCareCase.get_db().view( 'case_types_by_domain/view', startkey=[domain] + type_key, endkey=[domain] + type_key + [{}], reduce=False, include_docs=False, ) ]
def bulk_form_management_async(archive_or_restore, domain, couch_user, form_ids_or_query_string): # form_ids_or_query_string - can either be list of formids or a BulkFormMangementInterface query url def get_ids_from_url(query_string, domain, couch_user): from django.http import HttpRequest, QueryDict _request = HttpRequest() _request.couch_user = couch_user _request.user = couch_user.get_django_user() _request.domain = domain _request.couch_user.current_domain = domain _request.GET = QueryDict(query_string) dispatcher = EditDataInterfaceDispatcher() return dispatcher.dispatch( _request, render_as='form_ids', domain=domain, report_slug=BulkFormManagementInterface.slug, skip_permissions_check=True, ) task = bulk_form_management_async mode = FormManagementMode(archive_or_restore, validate=True) if type(form_ids_or_query_string) == list: xform_ids = form_ids_or_query_string elif isinstance(form_ids_or_query_string, basestring): xform_ids = get_ids_from_url(form_ids_or_query_string, domain, couch_user) if not xform_ids: soft_assert(notify_admins=True, exponential_backoff=False) return {'messages': {'errors': [_('No Forms are supplied')]}} response = archive_or_restore_forms(domain, couch_user, xform_ids, mode, task) return response
def get_case_from_case_update(self, case_update, xform): """ Gets or updates an existing case, based on a block of data in a submitted form. Doesn't save anything. Returns a CaseUpdateMetadata object. """ case = self.get(case_update.id) if case is None: if xform.metadata and xform.metadata.commcare_version: from distutils.version import LooseVersion commcare_version = xform.metadata.commcare_version message = "Case created without create block" send_to = None if commcare_version >= LooseVersion("2.44"): send_to = "{}@{}.com".format('skelly', 'dimagi') message += " in CC version >= 2.44" soft_assert(to=send_to)( case_update.creates_case(), message, { 'xform_id': xform.form_id, 'case_id': case_update.id, 'domain': xform.domain, 'version': str(commcare_version) } ) case = self.case_update_strategy.case_from_case_update(case_update, xform) self.set(case.case_id, case) return CaseUpdateMetadata(case, is_creation=True, previous_owner_id=None) else: previous_owner = case.owner_id self.case_update_strategy(case).update_from_case_update(case_update, xform, self.get_cached_forms()) return CaseUpdateMetadata(case, is_creation=False, previous_owner_id=previous_owner)
def get_case_ids_in_domain(domain, type=None): if type is None: type_keys = [[]] elif isinstance(type, (list, tuple)): soft_assert('skelly@{}'.format('dimagi.com'))( False, 'get_case_ids_in_domain called with typle / list arg for type') type_keys = [[t] for t in type] elif isinstance(type, six.string_types): soft_assert_type_text(type) type_keys = [[type]] else: raise ValueError( "Argument type should be a string, tuple, or None: {!r}".format( type)) return [ res['id'] for type_key in type_keys for res in CommCareCase.get_db().view( 'case_types_by_domain/view', startkey=[domain] + type_key, endkey=[domain] + type_key + [{}], reduce=False, include_docs=False, ) ]
def _handle_duplicate(new_doc): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id try: existing_doc = FormAccessors( new_doc.domain).get_with_attachments(conflict_id) except ResourceNotFound: # Original form processing failed but left behind a form doc with no # attachments. It's safe to delete this now since we're going to re-process # the form anyway. # https://sentry.io/dimagi/commcarehq/issues/261297321 from couchforms.models import XFormInstance XFormInstance.get_db().delete_doc(conflict_id) return FormProcessingResult(new_doc) existing_md5 = existing_doc.xml_md5() new_md5 = new_doc.xml_md5() if existing_md5 != new_md5: if new_doc.xmlns != existing_doc.xmlns: # if the XMLNS has changed this probably isn't a form edit # it could be a UUID clash (yes we've had that before) # Assign a new ID to the form and process as normal + notify_admins xform = interface.assign_new_id(new_doc) soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False)( False, "Potential UUID clash", { 'incoming_form_id': conflict_id, 'existing_form_id': existing_doc.form_id, 'new_form_id': xform.form_id, 'incoming_xmlns': new_doc.xmlns, 'existing_xmlns': existing_doc.xmlns, 'domain': new_doc.domain, }) return FormProcessingResult(xform) else: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface) return FormProcessingResult(new_doc, existing_doc) else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return FormProcessingResult(duplicate, existing_doc)
def __init__(self, fn, vary_on, cache): self.fn = fn self.cache = cache self.prefix = '{}.{}'.format( fn.__name__[:40] + (fn.__name__[40:] and '..'), self._hash(inspect.getsource(fn), 8) ) arg_names = inspect.getargspec(self.fn).args if not isfunction(vary_on): vary_on = [part.split('.') for part in vary_on] vary_on = [(part[0], tuple(part[1:])) for part in vary_on] for arg, attrs in vary_on: if arg not in arg_names: raise ValueError( 'We cannot vary on "{}" because the function {} has ' 'no such argument'.format(arg, self.fn.__name__) ) self.encoding_assert = soft_assert( to=['{}@{}'.format('skelly', 'dimagi.com')], fail_if_debug=False, skip_frames=5, ) self.vary_on = vary_on
def __init__(self, fn, vary_on, cache): self.fn = fn self.cache = cache self.prefix = '{}.{}'.format( fn.__name__[:40] + (fn.__name__[40:] and '..'), self._hash(inspect.getsource(fn), 8)) arg_names = inspect.getargspec(self.fn).args if not isfunction(vary_on): vary_on = [part.split('.') for part in vary_on] vary_on = [(part[0], tuple(part[1:])) for part in vary_on] for arg, attrs in vary_on: if arg not in arg_names: raise ValueError( 'We cannot vary on "{}" because the function {} has ' 'no such argument'.format(arg, self.fn.__name__)) self.encoding_assert = soft_assert( notify_admins=True, fail_if_debug=False, skip_frames=5, ) self.vary_on = vary_on
import base64 import hashlib import hmac import logging from functools import wraps from django.conf import settings from django.http import HttpResponse from corehq.util.soft_assert.api import soft_assert auth_logger = logging.getLogger("commcare_auth") _soft_assert = soft_assert(notify_admins=True) def convert_to_bytestring_if_unicode(shared_key): return shared_key.encode('utf-8') if isinstance(shared_key, str) else shared_key def get_hmac_digest(shared_key, data): hm = hmac.new(convert_to_bytestring_if_unicode(shared_key), convert_to_bytestring_if_unicode(data), hashlib.sha256) digest = base64.b64encode(hm.digest()) return digest.decode('utf-8') def validate_request_hmac(setting_name, ignore_if_debug=False, audit_user='******'): """ Decorator to validate request sender using a shared secret to compare the HMAC of the request body or query string with the value of the `X-MAC-DIGEST' header.
def _handle_duplicate(new_doc): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments :returns: A two-tuple: `(<new form>, <duplicate form or None>)` The new form may have a different `form_id` than `new_doc.form_id`. """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id try: existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id) except ResourceNotFound: # Original form processing failed but left behind a form doc with no # attachments. It's safe to delete this now since we're going to re-process # the form anyway. from couchforms.models import XFormInstance XFormInstance.get_db().delete_doc(conflict_id) return new_doc, None existing_md5 = existing_doc.xml_md5() new_md5 = new_doc.xml_md5() if existing_md5 != new_md5: _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False) if new_doc.xmlns != existing_doc.xmlns: # if the XMLNS has changed this probably isn't a form edit # it could be a UUID clash (yes we've had that before) # Assign a new ID to the form and process as normal + notify_admins xform = interface.assign_new_id(new_doc) _soft_assert( False, "Potential UUID clash", { 'incoming_form_id': conflict_id, 'existing_form_id': existing_doc.form_id, 'new_form_id': xform.form_id, 'incoming_xmlns': new_doc.xmlns, 'existing_xmlns': existing_doc.xmlns, 'domain': new_doc.domain, } ) return xform, None else: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface) device_id = new_doc.metadata.deviceID # we expect edits from formplayer so exclude those _soft_assert( device_id == FORMPLAYER_DEVICE_ID, "Form edit", { 'form_id': new_doc.form_id, 'deprecated_form': existing_doc.form_id, 'domain': new_doc.domain, 'device_id': device_id, 'cc_version': new_doc.metadata.appVersion, } ) return new_doc, existing_doc else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return duplicate, existing_doc
from __future__ import absolute_import from __future__ import unicode_literals import base64 import hashlib import hmac from functools import wraps import six from django.conf import settings from django.http import HttpResponse from corehq.util.soft_assert.api import soft_assert _soft_assert = soft_assert(notify_admins=True) def convert_to_bytestring_if_unicode(shared_key): return shared_key.encode('utf-8') if isinstance(shared_key, six.text_type) else shared_key def get_hmac_digest(shared_key, data): hm = hmac.new(convert_to_bytestring_if_unicode(shared_key), convert_to_bytestring_if_unicode(data), hashlib.sha256) digest = base64.b64encode(hm.digest()) return digest def validate_request_hmac(setting_name, ignore_if_debug=False): """ Decorator to validate request sender using a shared secret to compare the HMAC of the request body or query string with the value of the `X-MAC-DIGEST' header.
def test_message_newlines(self): _soft_assert = soft_assert(notify_admins=True) _soft_assert(False, "don't\ncrash")
import sys import os from couchdbkit import ResourceNotFound, ResourceConflict from django.conf import settings from django.core.management.base import BaseCommand, CommandError from http_parser.http import ParserError from restkit import RequestError from corehq.apps.domain.models import Domain from corehq.apps.domainsync.management.commands.copy_utils import copy_postgres_data_for_docs from corehq.util.dates import iso_string_to_date from dimagi.utils.couch.database import get_db, iter_docs from corehq.apps.domainsync.config import DocumentTransform, save from couchdbkit.client import Database from optparse import make_option from corehq.util.soft_assert.api import soft_assert _soft_assert = soft_assert('{}@{}'.format('tsheffels', 'dimagi.com')) # doctypes we want to be careful not to copy, which must be explicitly # specified with --include from dimagi.utils.parsing import json_format_date DEFAULT_EXCLUDE_TYPES = [ 'ReportNotification', 'WeeklyNotification', 'DailyNotification' ] NUM_PROCESSES = 8 class Command(BaseCommand):
def test_message_newlines(self): _soft_assert = soft_assert(notify_admins=True) _soft_assert(False, u"don't\ncrash")
from abc import ABCMeta, abstractmethod, abstractproperty import six from casexml.apps.case.exceptions import IllegalCaseId from corehq.util.soft_assert.api import soft_assert from dimagi.utils.couch import release_lock from corehq.form_processor.interfaces.processor import CaseUpdateMetadata, FormProcessorInterface _soft_assert = soft_assert(to="{}@{}.com".format('skelly', 'dimagi'), notify_admins=True) def _get_id_for_case(case): if isinstance(case, dict): return case['_id'] return case.case_id class AbstractCaseDbCache(six.with_metaclass(ABCMeta)): """ A temp object we use to keep a cache of in-memory cases around so we can get the latest updates even if they haven't been saved to the database. Also provides some type checking safety. This class can be used as a re-entrant context manager: with case_db: case_db.get('case1') with case_db: case_db.get('case2') """ @abstractproperty
def _handle_duplicate(new_doc): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments :returns: A two-tuple: `(<new form>, <duplicate form or None>)` The new form may have a different `form_id` than `new_doc.form_id`. """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id try: existing_doc = FormAccessors(new_doc.domain).get_with_attachments(conflict_id) except ResourceNotFound: # Original form processing failed but left behind a form doc with no # attachments. It's safe to delete this now since we're going to re-process # the form anyway. from couchforms.models import XFormInstance XFormInstance.get_db().delete_doc(conflict_id) return new_doc, None try: existing_md5 = existing_doc.xml_md5() except MissingFormXml: existing_md5 = None if not existing_doc.is_error: existing_doc.problem = 'Missing form.xml' new_md5 = new_doc.xml_md5() if existing_md5 != new_md5: _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False) if new_doc.xmlns != existing_doc.xmlns: # if the XMLNS has changed this probably isn't a form edit # it could be a UUID clash (yes we've had that before) # Assign a new ID to the form and process as normal + notify_admins xform = interface.assign_new_id(new_doc) _soft_assert( False, "Potential UUID clash", { 'incoming_form_id': conflict_id, 'existing_form_id': existing_doc.form_id, 'new_form_id': xform.form_id, 'incoming_xmlns': new_doc.xmlns, 'existing_xmlns': existing_doc.xmlns, 'domain': new_doc.domain, } ) return xform, None else: if existing_doc.is_error and not existing_doc.initial_processing_complete: # edge case from ICDS where a form errors and then future re-submissions of the same # form do not have the same MD5 hash due to a bug on mobile: # see https://dimagi-dev.atlassian.net/browse/ICDS-376 # since we have a new form and the old one was not successfully processed # we can effectively ignore this form and process the new one as normal if not interface.use_sql_domain: new_doc._rev, existing_doc._rev = existing_doc._rev, new_doc._rev interface.assign_new_id(existing_doc) existing_doc.save() return new_doc, None else: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID NotAllowed.check(new_doc.domain) existing_doc, new_doc = apply_deprecation(existing_doc, new_doc, interface) return new_doc, existing_doc else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return duplicate, existing_doc
def _handle_duplicate(new_doc): """ Handle duplicate xforms and xform editing ('deprecation') existing doc *must* be validated as an XFormInstance in the right domain and *must* include inline attachments :returns: A two-tuple: `(<new form>, <duplicate form or None>)` The new form may have a different `form_id` than `new_doc.form_id`. """ interface = FormProcessorInterface(new_doc.domain) conflict_id = new_doc.form_id existing_doc = XFormInstance.objects.get_with_attachments( conflict_id, new_doc.domain) is_icds = settings.SERVER_ENVIRONMENT in settings.ICDS_ENVS try: if is_icds and new_doc.metadata.deviceID == existing_doc.metadata.deviceID: # ICDS does not use 'edit form' functionality via the web and form editing is not possible # on mobile devices so it's safe to assume this is a duplicate without checking md5 etc. duplicate = interface.deduplicate_xform(new_doc) return duplicate, existing_doc existing_md5 = existing_doc.xml_md5() except MissingFormXml: existing_md5 = new_md5 = None if not existing_doc.is_error: existing_doc.problem = 'Missing form.xml' else: # only do this if we need to do the comparison new_md5 = new_doc.xml_md5() if existing_md5 is None or existing_md5 != new_md5: def _deprecate_old_form(): existing, new = apply_deprecation(existing_doc, new_doc, interface) return new, existing def _replace_old_form(): interface.assign_new_id(existing_doc) existing_doc.orig_id = new_doc.form_id existing_doc.save() return new_doc, None _soft_assert = soft_assert(to='{}@{}.com'.format('skelly', 'dimagi'), exponential_backoff=False) if new_doc.xmlns != existing_doc.xmlns: # if the XMLNS has changed this probably isn't a form edit # it could be a UUID clash (yes we've had that before) # Assign a new ID to the form and process as normal + notify_admins xform = interface.assign_new_id(new_doc) _soft_assert( False, "Potential UUID clash", { 'incoming_form_id': conflict_id, 'existing_form_id': existing_doc.form_id, 'new_form_id': xform.form_id, 'incoming_xmlns': new_doc.xmlns, 'existing_xmlns': existing_doc.xmlns, 'domain': new_doc.domain, }) return xform, None else: if existing_doc.is_error: # edge case from ICDS where a form errors and then future re-submissions of the same # form do not have the same MD5 hash due to a bug on mobile: # see https://dimagi-dev.atlassian.net/browse/ICDS-376 if not existing_doc.initial_processing_complete: # since we have a new form and the old one was not successfully processed # we can effectively ignore this form and process the new one as normal return _replace_old_form() elif not interface.form_has_case_transactions( existing_doc.form_id): # likely an error during saving return _replace_old_form() else: return _deprecate_old_form() else: # if the form contents are not the same: # - "Deprecate" the old form by making a new document with the same contents # but a different ID and a doc_type of XFormDeprecated # - Save the new instance to the previous document to preserve the ID return _deprecate_old_form() else: # follow standard dupe handling, which simply saves a copy of the form # but a new doc_id, and a doc_type of XFormDuplicate duplicate = interface.deduplicate_xform(new_doc) return duplicate, existing_doc