def _get_xform(xform_id):
db = XFormInstance.get_db()
doc = db.get(xform_id)
if doc['doc_type'] in doc_types():
return doc_types()[doc['doc_type']].wrap(doc)
if doc['doc_type'] == "%s%s" % (XFormInstance.__name__, DELETED_SUFFIX):
raise UnexpectedDeletedXForm(xform_id)
raise ResourceNotFound(xform_id)
def get_forms_by_type(domain, type_, recent_first=False,
limit=None):
assert type_ in doc_types()
# no production code should be pulling all forms in one go!
assert limit is not None
startkey = [domain, type_]
endkey = startkey + [{}]
if recent_first:
startkey, endkey = endkey, startkey
return XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False,
descending=recent_first,
include_docs=True,
limit=limit,
classes=doc_types(),
).all()
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
form_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
)
form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status(
"XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0))
has_diffs |= self._print_status(
'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only
)
case_ids_in_couch = set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
)
case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status(
'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short, diffs_only
)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(
key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only
)
if diffs_only and not has_diffs:
print(shell_green("No differences found between old and new docs!"))
return has_diffs
def get_number_of_forms_by_type(domain, type_):
assert type_ in doc_types()
startkey = [domain, type_]
endkey = startkey + [{}]
submissions = XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=True,
).one()
return submissions['value'] if submissions else 0
def get_forms_of_all_types(domain):
startkey = [domain]
endkey = startkey + [{}]
return XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False,
include_docs=True,
classes=doc_types(),
).all()
def _get_form_or_404(id):
# maybe this should be a more general utility a-la-django's get_object_or_404
try:
xform_json = XFormInstance.get_db().get(id)
except ResourceNotFound:
raise Http404()
doc_type = doc_types().get(xform_json.get('doc_type'))
if not doc_type:
raise Http404()
return doc_type.wrap(xform_json)
def get_number_of_forms_by_type(domain, type_):
# todo: this is only used to display totals in the "raw forms" report and can probably be removed
assert type_ in doc_types()
startkey = [domain, type_]
endkey = startkey + [{}]
submissions = XFormInstance.view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=True,
stale=stale_ok(),
).one()
return submissions['value'] if submissions else 0
def get_with_attachments(form_id):
doc = XFormInstance.get_db().get(form_id, attachments=True)
doc = doc_types()[doc['doc_type']].wrap(doc)
if doc.external_blobs:
for name, meta in doc.external_blobs.iteritems():
with doc.fetch_attachment(name, stream=True) as content:
doc.deferred_put_attachment(
content,
name,
content_type=meta.content_type,
content_length=meta.content_length,
)
return doc
def handle(self, domain, doc_type, **options):
csv = options.get('csv')
startdate = options.get('start')
enddate = options.get('end')
form_doc_types = doc_types()
if startdate or enddate:
if doc_type in CASE_DOC_TYPES or doc_type in form_doc_types:
if not should_use_sql_backend(domain):
raise CommandError("Date filtering not supported for Couch domains")
if startdate and enddate and enddate <= startdate:
raise CommandError("enddate must be after startdate")
handlers = {
'CommCareCase': compare_cases,
'CommCareCase-Deleted': compare_cases,
'CommCareUser': _compare_users,
'CommCareUser-Deleted': _compare_users,
'WebUser': _compare_users,
}
handlers.update({doc_type: compare_xforms for doc_type in form_doc_types})
try:
primary_count, es_count, primary_only, es_only = handlers[doc_type](domain, doc_type, startdate, enddate)
except KeyError:
raise CommandError('Unsupported doc type. Use on of: {}'.format(', '.join(handlers)))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([50, 50])
date_range_output = ''
if startdate or enddate:
end = (enddate or datetime.utcnow().date()).strftime(DATE_FORMAT)
start = startdate.strftime(DATE_FORMAT)
date_range_output = ' (Between {} and {})'.format(start, end)
print("\nDoc ID analysis for {}{}\n".format(doc_type, date_range_output))
print("Primary Count: {}".format(primary_count))
print("ES Count: {}\n".format(es_count))
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(
['Only in Primary', 'Only in ES'],
zip_longest(primary_only, es_only, fillvalue='')
)
def get_with_attachments(form_id):
doc = XFormInstance.get_db().get(form_id)
doc = doc_types()[doc['doc_type']].wrap(doc)
if doc.external_blobs:
for name, meta in six.iteritems(doc.external_blobs):
with doc.fetch_attachment(name, stream=True) as content:
doc.deferred_put_attachment(
content,
name,
content_type=meta.content_type,
content_length=meta.content_length,
)
else:
# xforms are expected to at least have the XML attachment
raise ResourceNotFound("XForm attachment missing: {}".format(form_id))
return doc
def _blow_away_migration(domain):
assert not should_use_sql_backend(domain)
delete_diff_db(domain)
for doc_type in doc_types():
sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
def get_form_ids_by_type(domain, type_, start=None, end=None):
assert type_ in doc_types()
startkey = [domain, type_]
if end:
endkey = startkey + end.isoformat()
else:
endkey = startkey + [{}]
if start:
startkey.append(start.isoformat())
return [row['id'] for row in XFormInstance.get_db().view(
"couchforms/all_submissions_by_domain",
startkey=startkey,
endkey=endkey,
reduce=False,
)]
def get_diff_stats(self, domain):
db = get_diff_db(domain)
diff_stats = db.get_diff_stats()
stats = {}
def _update_stats(doc_type, couch_count, sql_count):
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
if diff_count or couch_count != sql_count:
stats[doc_type] = (couch_count, sql_count, diff_count, num_docs_with_diffs)
for doc_type in doc_types():
form_ids_in_couch = len(set(get_form_ids_by_type(domain, doc_type)))
form_ids_in_sql = len(set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)))
_update_stats(doc_type, form_ids_in_couch, form_ids_in_sql)
form_ids_in_couch = len(set(get_doc_ids_in_domain_by_type(
domain, "XFormInstance-Deleted", XFormInstance.get_db())
))
form_ids_in_sql = len(set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)))
_update_stats("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql)
case_ids_in_couch = len(set(get_case_ids_in_domain(domain)))
case_ids_in_sql = len(set(CaseAccessorSQL.get_case_ids_in_domain(domain)))
_update_stats("CommCareCase", case_ids_in_couch, case_ids_in_sql)
if self.strict:
# only care about these in strict mode
case_ids_in_couch = len(set(get_doc_ids_in_domain_by_type(
domain, "CommCareCase-Deleted", XFormInstance.get_db())
))
case_ids_in_sql = len(set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)))
_update_stats("CommCareCase-Deleted", case_ids_in_couch, case_ids_in_sql)
if diff_stats:
for key in diff_stats.keys():
_update_stats(key, 0, 0)
return stats
def get_forms_by_id(form_ids):
forms = iter_docs(XFormInstance.get_db(), form_ids)
doctypes = doc_types()
return [doctypes[form['doc_type']].wrap(form) for form in forms if form.get('doc_type') in doctypes]
def is_duplicate(cls, xform_id, domain=None):
if domain:
try:
existing_doc = XFormInstance.get_db().get(xform_id)
except ResourceNotFound:
return False
return existing_doc.get('domain') == domain and existing_doc.get('doc_type') in doc_types()
else:
return xform_id in XFormInstance.get_db()
def test_wrappable(self):
types = couchforms_models.all_known_formlike_doc_types()
for key in couchforms_models.doc_types().keys():
self.assertTrue(key in types)
def get_doc_info(doc, domain_hint=None, cache=None):
"""
cache is just a dictionary that you can keep passing in to speed up info
retrieval.
"""
domain = doc.get("domain") or domain_hint
doc_type = doc.get("doc_type")
doc_id = doc.get("_id")
generic_delete = doc_type.endswith(DELETED_SUFFIX)
def has_doc_type(doc_type, expected_doc_type):
return doc_type == expected_doc_type or doc_type == ("%s%s" % (expected_doc_type, DELETED_SUFFIX))
if domain_hint and not (doc.get("domain") == domain_hint or domain_hint in doc.get("domains", ())):
raise DomainMismatchException("Doc '%s' does not match the domain_hint '%s'" % (doc_id, domain_hint))
if cache and doc_id in cache:
return cache[doc_id]
if has_doc_type(doc_type, "Application") or has_doc_type(doc_type, "RemoteApp"):
if doc.get("copy_of"):
doc_info = DocInfo(
display=u"%s (#%s)" % (doc["name"], doc["version"]),
type_display=_("Application Build"),
link=reverse("corehq.apps.app_manager.views.download_index", args=[domain, doc_id]),
is_deleted=generic_delete,
)
else:
doc_info = DocInfo(
display=doc["name"],
type_display=_("Application"),
link=reverse("corehq.apps.app_manager.views.view_app", args=[domain, doc_id]),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, "CommCareCaseGroup"):
from corehq.apps.data_interfaces.views import CaseGroupCaseManagementView
doc_info = DocInfo(
type_display=_("Case Group"),
display=doc["name"],
link=reverse(CaseGroupCaseManagementView.urlname, args=[domain, doc_id]),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, "CommCareCase"):
doc_info = DocInfo(
display=doc["name"],
type_display=_("Case"),
link=reverse("case_details", args=[domain, doc_id]),
is_deleted=generic_delete,
)
elif any([has_doc_type(doc_type, d) for d in couchforms_models.doc_types().keys()]):
doc_info = DocInfo(
type_display=_("Form"), link=reverse("render_form_data", args=[domain, doc_id]), is_deleted=generic_delete
)
elif doc_type in ("CommCareUser",):
doc_info = DocInfo(
display=raw_username(doc["username"]),
type_display=_("Mobile Worker"),
link=reverse("edit_commcare_user", args=[domain, doc_id]),
is_deleted=doc.get("base_doc", "").endswith(DELETED_SUFFIX),
)
elif doc_type in ("WebUser",):
doc_info = DocInfo(
type_display=_("Web User"),
display=doc["username"],
link=reverse("user_account", args=[domain, doc_id]),
is_deleted=doc.get("base_doc", "").endswith(DELETED_SUFFIX),
)
elif has_doc_type(doc_type, "Group"):
from corehq.apps.users.views.mobile import EditGroupMembersView
doc_info = DocInfo(
type_display=_("Group"),
display=doc["name"],
link=reverse(EditGroupMembersView.urlname, args=[domain, doc_id]),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, "Domain"):
if doc["is_snapshot"] and doc["published"]:
urlname = "project_info"
else:
urlname = "domain_basic_info"
doc_info = DocInfo(
type_display=_("Domain"),
display=doc["name"],
link=reverse(urlname, kwargs={"domain": doc["name"]}),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, "Location"):
from corehq.apps.locations.views import EditLocationView
doc_info = DocInfo(
type_display=_("Location"),
display=doc["name"],
link=reverse(EditLocationView.urlname, args=[domain, doc_id]),
is_deleted=generic_delete,
)
else:
doc_info = DocInfo(is_deleted=generic_delete)
doc_info.id = doc_id
doc_info.domain = domain
doc_info.type = doc_type
if cache:
cache[doc_id] = doc_info
return doc_info
from corehq.apps.api.resources import v0_1, v0_3, HqBaseResource, DomainSpecificResourceMixin, \
SimpleSortableResourceMixin
from corehq.apps.api.es import XFormES, CaseES, ESQuerySet, es_search
from corehq.apps.api.fields import ToManyDocumentsField, UseIfRequested, ToManyDictField, ToManyListDictField
from corehq.apps.api.serializers import CommCareCaseSerializer
from no_exceptions.exceptions import Http400
# By the time a test case is running, the resource is already instantiated,
# so as a hack until this can be remedied, there is a global that
# can be set to provide a mock.
MOCK_XFORM_ES = None
MOCK_CASE_ES = None
xform_doc_types = doc_types()
class XFormInstanceResource(SimpleSortableResourceMixin, v0_3.XFormInstanceResource, DomainSpecificResourceMixin):
# Some fields that were present when just fetching individual docs are
# not present for e.g. devicelogs and must be allowed blank
uiversion = fields.CharField(attribute='uiversion', blank=True, null=True)
metadata = fields.DictField(attribute='metadata', blank=True, null=True)
domain = fields.CharField(attribute='domain')
app_id = fields.CharField(attribute='app_id', null=True)
build_id = fields.CharField(attribute='build_id', null=True)
initial_processing_complete = fields.BooleanField(
attribute='initial_processing_complete', null=True)
problem = fields.CharField(attribute='problem', null=True)
def test_wrappable(self):
types = couchforms_models.all_known_formlike_doc_types()
for key in couchforms_models.doc_types().keys():
self.assertTrue(key in types)
def is_duplicate(cls, xform_id, domain=None):
if domain:
try:
existing_doc = XFormInstance.get_db().get(xform_id)
except ResourceNotFound:
return False
return existing_doc.get('domain') == domain and existing_doc.get('doc_type') in doc_types()
else:
return xform_id in XFormInstance.get_db()
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(
domain, status))
db = get_diff_db(domain)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"CommCareCase", (0, 0))
has_diffs |= self._print_status('CommCareCase', case_ids_in_couch,
case_ids_in_sql, diff_count,
num_docs_with_diffs, short, diffs_only)
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"CommCareCase-Deleted", (0, 0))
has_diffs |= self._print_status('CommCareCase-Deleted',
case_ids_in_couch, case_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
def get_doc_info(doc, domain_hint=None, cache=None):
"""
cache is just a dictionary that you can keep passing in to speed up info
retrieval.
"""
domain = doc.get('domain') or domain_hint
doc_type = doc.get('doc_type')
doc_id = doc.get('_id')
generic_delete = doc_type.endswith(DELETED_SUFFIX)
def has_doc_type(doc_type, expected_doc_type):
return (doc_type == expected_doc_type or
doc_type == ('%s%s' % (expected_doc_type, DELETED_SUFFIX)))
if (
domain_hint and
not (
doc.get('domain') == domain_hint or
domain_hint in doc.get('domains', ())
)
):
raise DomainMismatchException("Doc '%s' does not match the domain_hint '%s'" % (doc_id, domain_hint))
if cache and doc_id in cache:
return cache[doc_id]
if has_doc_type(doc_type, 'Application') or has_doc_type(doc_type, 'RemoteApp'):
if doc.get('copy_of'):
doc_info = DocInfo(
display=u'%s (#%s)' % (doc['name'], doc['version']),
type_display=_('Application Build'),
link=reverse(
'download_index',
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
else:
doc_info = DocInfo(
display=doc['name'],
type_display=_('Application'),
link=reverse(
'view_app',
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, 'CommCareCaseGroup'):
from corehq.apps.data_interfaces.views import CaseGroupCaseManagementView
doc_info = DocInfo(
type_display=_('Case Group'),
display=doc['name'],
link=reverse(
CaseGroupCaseManagementView.urlname,
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, 'CommCareCase'):
doc_info = case_docinfo(domain, doc_id, doc['name'], generic_delete)
elif any([has_doc_type(doc_type, d) for d in couchforms_models.doc_types().keys()]):
doc_info = form_docinfo(domain, doc_id, generic_delete)
elif doc_type in ('CommCareUser',):
doc_info = DocInfo(
display=raw_username(doc['username']),
type_display=_('Mobile Worker'),
link=get_commcareuser_url(domain, doc_id),
is_deleted=doc.get('base_doc', '').endswith(DELETED_SUFFIX),
)
elif doc_type in ('WebUser',):
doc_info = DocInfo(
type_display=_('Web User'),
display=doc['username'],
link=get_webuser_url(domain, doc_id),
is_deleted=doc.get('base_doc', '').endswith(DELETED_SUFFIX),
)
elif has_doc_type(doc_type, 'Group'):
from corehq.apps.users.views.mobile import EditGroupMembersView
doc_info = DocInfo(
type_display=_('Group'),
display=doc['name'],
link=reverse(
EditGroupMembersView.urlname,
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, 'Domain'):
if doc['is_snapshot'] and doc['published']:
urlname = 'project_info'
else:
urlname = 'domain_basic_info'
doc_info = DocInfo(
type_display=_('Domain'),
display=doc['name'],
link=reverse(
urlname,
kwargs={'domain' : doc['name']}
),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, 'Location'):
from corehq.apps.locations.views import EditLocationView
doc_info = DocInfo(
type_display=_('Location'),
display=doc['name'],
link=reverse(
EditLocationView.urlname,
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
else:
doc_info = DocInfo(
is_deleted=generic_delete,
)
doc_info.id = doc_id
doc_info.domain = domain
doc_info.type = doc_type
if cache:
cache[doc_id] = doc_info
return doc_info
from corehq.apps.users.util import format_username
from corehq.motech.repeaters.models import Repeater
from corehq.motech.repeaters.utils import get_all_repeater_types
from corehq.util.view_utils import absolute_reverse
from couchforms.models import doc_types
from custom.hope.models import HOPECase, CC_BIHAR_NEWBORN, CC_BIHAR_PREGNANCY
from no_exceptions.exceptions import Http400
import six
# By the time a test case is running, the resource is already instantiated,
# so as a hack until this can be remedied, there is a global that
# can be set to provide a mock.
MOCK_XFORM_ES = None
MOCK_CASE_ES = None
xform_doc_types = doc_types()
class XFormInstanceResource(SimpleSortableResourceMixin, HqBaseResource,
DomainSpecificResourceMixin):
"""This version of the form resource is built of Elasticsearch data
which gets wrapped by ``ESXFormInstance``.
No type conversion is done e.g. dates and some fields are named differently than in the
Python models.
"""
id = fields.CharField(attribute='_id', readonly=True, unique=True)
domain = fields.CharField(attribute='domain')
form = fields.DictField(attribute='form_data')
type = fields.CharField(attribute='type')
def should_handle_as_duplicate_or_edit(cls, xform_id, domain):
existing_doc = XFormInstance.get_db().get(xform_id)
return existing_doc.get('domain') == domain and existing_doc.get('doc_type') in doc_types()
def get_doc_info(doc, domain_hint=None, cache=None):
"""
cache is just a dictionary that you can keep passing in to speed up info
retrieval.
"""
domain = doc.get('domain') or domain_hint
doc_type = doc.get('doc_type')
doc_id = doc.get('_id')
assert doc.get('domain') == domain or domain in doc.get('domains', ())
if cache and doc_id in cache:
return cache[doc_id]
if doc_type in ('Application', 'RemoteApp'):
if doc.get('copy_of'):
doc_info = DocInfo(
display=u'%s (#%s)' % (doc['name'], doc['version']),
type_display=_('Application Build'),
link=reverse(
'corehq.apps.app_manager.views.download_index',
args=[domain, doc_id],
),
)
else:
doc_info = DocInfo(
display=doc['name'],
type_display=_('Application'),
link=reverse(
'corehq.apps.app_manager.views.view_app',
args=[domain, doc_id],
),
)
elif doc_type in ('CommCareCase',):
doc_info = DocInfo(
display=doc['name'],
type_display=_('Case'),
link=reverse(
'case_details',
args=[domain, doc_id],
),
)
elif doc_type in (couchforms_models.doc_types().keys()):
doc_info = DocInfo(
type_display=_('Form'),
link=reverse(
'render_form_data',
args=[domain, doc_id],
),
)
elif doc_type in ('CommCareUser',):
doc_info = DocInfo(
display=raw_username(doc['username']),
type_display=_('Mobile Worker'),
link=reverse(
'edit_commcare_user',
args=[domain, doc_id],
),
)
elif doc_type in ('WebUser',):
doc_info = DocInfo(
type_display=_('Web User'),
display=doc['username'],
link=reverse(
'user_account',
args=[domain, doc_id],
),
)
elif doc_type in ('Group',):
from corehq.apps.users.views.mobile import EditGroupMembersView
doc_info = DocInfo(
type_display=_('Group'),
display=doc['name'],
link=reverse(
EditGroupMembersView.urlname,
args=[domain, doc_id],
),
)
elif doc_type in ('Domain',):
if doc['is_snapshot'] and doc['published']:
urlname = 'project_info'
else:
urlname = 'domain_basic_info'
doc_info = DocInfo(
type_display=_('Domain'),
display=doc['name'],
link=reverse(
urlname,
kwargs={'domain' : doc['name']}
),
)
elif doc_type == 'Location':
from corehq.apps.locations.views import EditLocationView
doc_info = DocInfo(
type_display=doc['location_type'],
display=doc['name'],
link=reverse(
EditLocationView.urlname,
args=[domain, doc_id],
),
)
else:
doc_info = DocInfo()
doc_info.id = doc_id
doc_info.domain = domain
doc_info.type = doc_type
if cache:
cache[doc_id] = doc_info
return doc_info
def get_with_attachments(form_id):
doc = XFormInstance.get_db().get(form_id, attachments=True)
return doc_types()[doc['doc_type']].wrap(doc)
def print_stats(self, domain, short=True, diffs_only=False):
status = get_couch_sql_migration_status(domain)
print("Couch to SQL migration status for {}: {}".format(
domain, status))
db = open_state_db(domain, self.state_dir)
try:
diff_stats = db.get_diff_stats()
except OperationalError:
diff_stats = {}
has_diffs = False
for doc_type in doc_types():
form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type))
if doc_type == "XFormInstance":
form_ids_in_couch.update(
get_doc_ids_in_domain_by_type(domain, "HQSubmission",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, doc_type))
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(doc_type, form_ids_in_couch,
form_ids_in_sql, diff_count,
num_docs_with_diffs, short,
diffs_only)
form_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted",
XFormInstance.get_db()))
form_ids_in_sql = set(
FormAccessorSQL.get_deleted_form_ids_in_domain(domain))
diff_count, num_docs_with_diffs = diff_stats.pop(
"XFormInstance-Deleted", (0, 0))
has_diffs |= self._print_status("XFormInstance-Deleted",
form_ids_in_couch, form_ids_in_sql,
diff_count, num_docs_with_diffs, short,
diffs_only)
ZERO = Counts(0, 0)
if db.has_doc_counts():
doc_counts = db.get_doc_counts()
couch_missing_cases = doc_counts.get("CommCareCase-couch",
ZERO).missing
else:
doc_counts = None
couch_missing_cases = 0
for doc_type in CASE_DOC_TYPES:
if doc_counts is not None:
counts = doc_counts.get(doc_type, ZERO)
case_ids_in_couch = db.get_missing_doc_ids(
doc_type) if counts.missing else set()
case_ids_in_sql = counts
elif doc_type == "CommCareCase":
case_ids_in_couch = set(get_case_ids_in_domain(domain))
case_ids_in_sql = set(
CaseAccessorSQL.get_case_ids_in_domain(domain))
elif doc_type == "CommCareCase-Deleted":
case_ids_in_couch = set(
get_doc_ids_in_domain_by_type(domain,
"CommCareCase-Deleted",
XFormInstance.get_db()))
case_ids_in_sql = set(
CaseAccessorSQL.get_deleted_case_ids_in_domain(domain))
else:
raise NotImplementedError(doc_type)
diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0))
has_diffs |= self._print_status(
doc_type,
case_ids_in_couch,
case_ids_in_sql,
diff_count,
num_docs_with_diffs,
short,
diffs_only,
)
if doc_type == "CommCareCase" and couch_missing_cases:
has_diffs = True
print(
shell_red("%s cases could not be loaded from Couch" %
couch_missing_cases))
if not short:
for case_id in db.get_missing_doc_ids(
"CommCareCase-couch"):
print(case_id)
if diff_stats:
for key, counts in diff_stats.items():
diff_count, num_docs_with_diffs = counts
has_diffs |= self._print_status(key, set(), set(), diff_count,
num_docs_with_diffs, short,
diffs_only)
if diffs_only and not has_diffs:
print(
shell_green("No differences found between old and new docs!"))
return has_diffs
def get_doc_info(doc, domain_hint=None, cache=None):
"""
cache is just a dictionary that you can keep passing in to speed up info
retrieval.
"""
domain = doc.get('domain') or domain_hint
doc_type = doc.get('doc_type')
doc_id = doc.get('_id')
generic_delete = doc_type.endswith(DELETED_SUFFIX)
def has_doc_type(doc_type, expected_doc_type):
return (doc_type == expected_doc_type or
doc_type == ('%s%s' % (expected_doc_type, DELETED_SUFFIX)))
if (
domain_hint and
not (
doc.get('domain') == domain_hint or
domain_hint in doc.get('domains', ())
)
):
raise DomainMismatchException("Doc '%s' does not match the domain_hint '%s'" % (doc_id, domain_hint))
if cache and doc_id in cache:
return cache[doc_id]
if has_doc_type(doc_type, 'Application') or has_doc_type(doc_type, 'RemoteApp'):
if doc.get('copy_of'):
doc_info = DocInfo(
display='%s (#%s)' % (doc['name'], doc['version']),
type_display=_('Application Build'),
link=reverse(
'download_index',
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
else:
doc_info = DocInfo(
display=doc['name'],
type_display=_('Application'),
link=reverse(
'view_app',
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, 'CommCareCaseGroup'):
from corehq.apps.data_interfaces.views import CaseGroupCaseManagementView
doc_info = DocInfo(
type_display=_('Case Group'),
display=doc['name'],
link=reverse(
CaseGroupCaseManagementView.urlname,
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, 'CommCareCase'):
doc_info = case_docinfo(domain, doc_id, doc['name'], generic_delete)
elif any([has_doc_type(doc_type, d) for d in couchforms_models.doc_types().keys()]):
doc_info = form_docinfo(domain, doc_id, generic_delete)
elif doc_type in ('CommCareUser',):
doc_info = DocInfo(
display=raw_username(doc['username']),
type_display=_('Mobile Worker'),
link=get_commcareuser_url(domain, doc_id),
is_deleted=doc.get('base_doc', '').endswith(DELETED_SUFFIX),
)
elif doc_type in ('WebUser',):
doc_info = DocInfo(
type_display=_('Web User'),
display=doc['username'],
link=get_webuser_url(domain, doc_id),
is_deleted=doc.get('base_doc', '').endswith(DELETED_SUFFIX),
)
elif has_doc_type(doc_type, 'Group'):
from corehq.apps.users.views.mobile import EditGroupMembersView
doc_info = DocInfo(
type_display=_('Group'),
display=doc['name'],
link=reverse(
EditGroupMembersView.urlname,
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, 'Domain'):
if doc['is_snapshot'] and doc['published']:
urlname = 'project_info'
else:
urlname = 'domain_basic_info'
doc_info = DocInfo(
type_display=_('Domain'),
display=doc['name'],
link=reverse(
urlname,
kwargs={'domain' : doc['name']}
),
is_deleted=generic_delete,
)
elif has_doc_type(doc_type, 'Location'):
from corehq.apps.locations.views import EditLocationView
doc_info = DocInfo(
type_display=_('Location'),
display=doc['name'],
link=reverse(
EditLocationView.urlname,
args=[domain, doc_id],
),
is_deleted=generic_delete,
)
else:
doc_info = DocInfo(
is_deleted=generic_delete,
)
doc_info.id = doc_id
doc_info.domain = domain
doc_info.type = doc_type
if cache:
cache[doc_id] = doc_info
return doc_info
