def _get_document_subtype(document_or_none):
type = _get_document_type(document_or_none)
if type in ('CommCareCase', 'CommCareCase-Deleted'):
return document_or_none.get('type', None)
elif type in all_known_formlike_doc_types():
return document_or_none.get('xmlns', None)
return None
def get_document_store_for_doc_type(domain, doc_type, case_type_or_xmlns=None, load_source="unknown"):
"""Only applies to documents that have a document type:
* forms
* cases
* locations
* all couch models
"""
from corehq.apps.change_feed import document_types
if doc_type in all_known_formlike_doc_types():
store = ReadonlyFormDocumentStore(domain, xmlns=case_type_or_xmlns)
load_counter = form_load_counter
elif doc_type in document_types.CASE_DOC_TYPES:
store = ReadonlyCaseDocumentStore(domain, case_type=case_type_or_xmlns)
load_counter = case_load_counter
elif doc_type == LOCATION_DOC_TYPE:
return ReadonlyLocationDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(
couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type
)
track_load = load_counter(load_source, domain)
return DocStoreLoadTracker(store, track_load)
def get_topic_for_doc_type(doc_type, data_source_type=None):
from corehq.apps.change_feed import document_types
from corehq.apps.locations.document_store import LOCATION_DOC_TYPE
if doc_type in document_types.CASE_DOC_TYPES:
return {
'sql': CASE_SQL,
'couch': CASE
}.get(data_source_type, CASE)
elif doc_type in all_known_formlike_doc_types():
return {
'sql': FORM_SQL,
'couch': FORM
}.get(data_source_type, FORM)
elif doc_type in document_types.DOMAIN_DOC_TYPES:
return DOMAIN
elif doc_type in document_types.MOBILE_USER_DOC_TYPES:
return COMMCARE_USER
elif doc_type in document_types.WEB_USER_DOC_TYPES:
return WEB_USER
elif doc_type in document_types.GROUP_DOC_TYPES:
return GROUP
elif doc_type in document_types.SYNCLOG_DOC_TYPES:
return SYNCLOG_SQL
elif doc_type in app_doc_types():
return APP
elif doc_type == LOCATION_DOC_TYPE:
return LOCATION
elif doc_type in ALL: # for docs that don't have a doc_type we use the Kafka topic
return doc_type
else:
# at some point we may want to make this more granular
return META # note this does not map to the 'meta' Couch database
def get_topic_for_doc_type(doc_type,
data_source_type=None,
default_topic=None):
from corehq.apps.change_feed import document_types
from corehq.apps.locations.document_store import LOCATION_DOC_TYPE
if doc_type in document_types.CASE_DOC_TYPES:
return {'sql': CASE_SQL, 'couch': CASE}.get(data_source_type, CASE)
elif doc_type in all_known_formlike_doc_types():
return {'sql': FORM_SQL, 'couch': FORM}.get(data_source_type, FORM)
elif doc_type in document_types.DOMAIN_DOC_TYPES:
return DOMAIN
elif doc_type in document_types.MOBILE_USER_DOC_TYPES:
return COMMCARE_USER
elif doc_type in document_types.WEB_USER_DOC_TYPES:
return WEB_USER
elif doc_type in document_types.GROUP_DOC_TYPES:
return GROUP
elif doc_type in document_types.SYNCLOG_DOC_TYPES:
return SYNCLOG_SQL
elif doc_type in app_doc_types():
return APP
elif doc_type == LOCATION_DOC_TYPE:
return LOCATION
elif doc_type in ALL: # for docs that don't have a doc_type we use the Kafka topic
return doc_type
elif default_topic:
return default_topic
else:
# at some point we may want to make this more granular
return META # note this does not map to the 'meta' Couch database
def test_change_provider_empty(self):
provider = CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[],
doc_types=all_known_formlike_doc_types())
self.assertEqual([],
[change for change in provider.iter_all_changes()])
def test_change_provider_empty(self):
provider = CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[],
doc_types=all_known_formlike_doc_types()
)
self.assertEqual([], [change for change in provider.iter_all_changes()])
def get_document_store_for_doc_type(domain,
doc_type,
case_type_or_xmlns=None,
load_source="unknown"):
"""Only applies to documents that have a document type:
* forms
* cases
* locations
* leddgers (V2 only)
* all couch models
"""
from corehq.apps.change_feed import document_types
if doc_type in all_known_formlike_doc_types():
store = FormDocumentStore(domain, xmlns=case_type_or_xmlns)
load_counter = form_load_counter
elif doc_type in document_types.CASE_DOC_TYPES:
store = CaseDocumentStore(domain, case_type=case_type_or_xmlns)
load_counter = case_load_counter
elif doc_type == LOCATION_DOC_TYPE:
return LocationDocumentStore(domain)
elif doc_type == topics.LEDGER:
return LedgerV2DocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type)
track_load = load_counter(load_source, domain)
return DocStoreLoadTracker(store, track_load)
def test_change_provider(self):
provider = CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=self.domains,
doc_types=all_known_formlike_doc_types())
doc_ids = {change.id for change in provider.iter_all_changes()}
self.assertEqual(doc_ids,
set(itertools.chain(*self.form_ids.values())))
def test_change_provider(self):
provider = CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=self.domains,
doc_types=all_known_formlike_doc_types()
)
doc_ids = {change.id for change in provider.iter_all_changes()}
self.assertEqual(doc_ids, set(itertools.chain(*list(self.form_ids.values()))))
def get_topic(document_type):
if document_type in ('CommCareCase', 'CommCareCase-Deleted'):
return CASE
elif document_type in all_known_formlike_doc_types():
return FORM
else:
# at some point we may want to make this more granular
return META
def _make_document_type(raw_doc_type, document):
if raw_doc_type in CASE_DOC_TYPES:
return _case_doc_type_constructor(raw_doc_type, document)
elif raw_doc_type in all_known_formlike_doc_types():
return _form_doc_type_constructor(raw_doc_type, document)
elif raw_doc_type in DOMAIN_DOC_TYPES:
return _domain_doc_type_constructor(raw_doc_type, document)
else:
return DocumentMetadata(raw_doc_type, None, _get_domain(document),
is_deletion(raw_doc_type))
def get_domain_form_change_provider(domains):
sql_domains = {domain for domain in domains if should_use_sql_backend(domain)}
couch_domains = set(domains) - sql_domains
return CompositeChangeProvider([
SqlDomainXFormChangeProvider(sql_domains),
CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=couch_domains,
doc_types=all_known_formlike_doc_types()
),
])
def get_domain_form_change_provider(domains):
sql_domains = {
domain
for domain in domains if should_use_sql_backend(domain)
}
couch_domains = set(domains) - sql_domains
return CompositeChangeProvider([
SqlDomainXFormChangeProvider(sql_domains),
CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=couch_domains,
doc_types=all_known_formlike_doc_types()),
])
def _get_primary_type(raw_doc_type):
if raw_doc_type in ('CommCareCase', 'CommCareCase-Deleted'):
return CASE
elif raw_doc_type in all_known_formlike_doc_types():
return FORM
elif raw_doc_type in ('Domain', 'Domain-Deleted', 'Domain-DUPLICATE'):
return DOMAIN
elif raw_doc_type in ('CommCareUser', 'CommCareUser-Deleted'):
return COMMCARE_USER
elif raw_doc_type in ('WebUser', 'WebUser-Deleted'):
return WEB_USER
elif raw_doc_type in ('Group', 'Group-Deleted'):
return GROUP
elif raw_doc_type in app_doc_types():
return APP
else:
# at some point we may want to make this more granular
return META
def _get_primary_type(raw_doc_type):
if raw_doc_type in ('CommCareCase', 'CommCareCase-Deleted'):
return CASE
elif raw_doc_type in all_known_formlike_doc_types():
return FORM
elif raw_doc_type in ('Domain', 'Domain-Deleted', 'Domain-DUPLICATE'):
return DOMAIN
elif raw_doc_type in ('CommCareUser', 'CommCareUser-Deleted'):
return COMMCARE_USER
elif raw_doc_type in ('WebUser', 'WebUser-Deleted'):
return WEB_USER
elif raw_doc_type in ('Group', 'Group-Deleted'):
return GROUP
elif raw_doc_type in app_doc_types():
return APP
else:
# at some point we may want to make this more granular
return META
def get_document_store_for_doc_type(domain, doc_type, case_type_or_xmlns=None):
"""Only applies to documents that have a document type:
* forms
* cases
* locations
* all couch models
"""
from corehq.apps.change_feed import document_types
if doc_type in all_known_formlike_doc_types():
return ReadonlyFormDocumentStore(domain, xmlns=case_type_or_xmlns)
elif doc_type in document_types.CASE_DOC_TYPES:
return ReadonlyCaseDocumentStore(domain, case_type=case_type_or_xmlns)
elif doc_type == LOCATION_DOC_TYPE:
return ReadonlyLocationDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type)
def get_document_store_for_doc_type(domain, doc_type, case_type_or_xmlns=None):
"""Only applies to documents that have a document type:
* forms
* cases
* locations
* all couch models
"""
from corehq.apps.change_feed import document_types
if doc_type in all_known_formlike_doc_types():
return ReadonlyFormDocumentStore(domain, xmlns=case_type_or_xmlns)
elif doc_type in document_types.CASE_DOC_TYPES:
return ReadonlyCaseDocumentStore(domain, case_type=case_type_or_xmlns)
elif doc_type == LOCATION_DOC_TYPE:
return ReadonlyLocationDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(
couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type
)
def delete_all_xforms(cls, domain=None):
logger.debug("Deleting all Couch xforms for domain %s", domain)
cls._delete_all(XFormInstance.get_db(), all_known_formlike_doc_types(), domain)
FormProcessorTestUtils.delete_all_sql_forms(domain)
def test_deleted(self):
self.assertTrue('XFormInstance-Deleted' in couchforms_models.all_known_formlike_doc_types())
def test_wrappable(self):
types = couchforms_models.all_known_formlike_doc_types()
for key in couchforms_models.doc_types().keys():
self.assertTrue(key in types)
def _get_couch_forms_by_doc_type(domain):
return _get_couch_doc_counts(CommCareCase.get_db(), domain,
all_known_formlike_doc_types())
def test_deleted(self):
self.assertTrue('XFormInstance-Deleted' in
couchforms_models.all_known_formlike_doc_types())
def test_wrappable(self):
types = couchforms_models.all_known_formlike_doc_types()
for key in couchforms_models.doc_types().keys():
self.assertTrue(key in types)
from .asyncforms import AsyncFormProcessor
from .casediff import CaseDiffProcess, CaseDiffQueue, NoCaseDiff
from .json2xml import convert_form_to_xml
from .statedb import init_state_db
from .staterebuilder import iter_unmigrated_docs
from .system_action import do_system_action
from .util import get_ids_from_string_or_file, exit_on_error, str_to_datetime
log = logging.getLogger(__name__)
CASE_DOC_TYPES = [
'CommCareCase',
'CommCareCase-Deleted',
]
UNPROCESSED_DOC_TYPES = list(all_known_formlike_doc_types() -
{'XFormInstance'})
def setup_logging(log_dir, slug, debug=False):
if debug:
assert log.level <= logging.DEBUG, log.level
logging.root.setLevel(logging.DEBUG)
for handler in logging.root.handlers:
if handler.name in ["file", "console"]:
handler.setLevel(logging.DEBUG)
if not log_dir:
return
time = datetime.utcnow().strftime("%Y%m%d%H%M%S")
log_file = os.path.join(log_dir, f"couch2sql-form-case-{time}-{slug}.log")
formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
def python_filter(self, change):
# designed to exactly mimic the behavior of couchforms/filters/xforms.js
doc = change.document
return doc.get('doc_type', None) in all_known_formlike_doc_types() and not is_device_report(doc)
from dimagi.utils.couch.undo import DELETED_SUFFIX
from dimagi.utils.chunked import chunked
from pillowtop.reindexer.change_providers.couch import CouchDomainDocTypeChangeProvider
from gevent.pool import Pool
import gevent
import six
import logging
from collections import defaultdict, deque
from corehq.util import cache_utils
from django_redis import get_redis_connection
_logger = logging.getLogger('main_couch_sql_datamigration')
CASE_DOC_TYPES = ['CommCareCase', 'CommCareCase-Deleted', ]
UNPROCESSED_DOC_TYPES = list(all_known_formlike_doc_types() - {'XFormInstance'})
def do_couch_to_sql_migration(domain, with_progress=True, debug=False, run_timestamp=None):
set_local_domain_sql_backend_override(domain)
CouchSqlDomainMigrator(
domain,
with_progress=with_progress,
debug=debug,
run_timestamp=run_timestamp
).migrate()
class CouchSqlDomainMigrator(object):
def __init__(self, domain, with_progress=True, debug=False, run_timestamp=None):
from corehq.apps.tzmigration.planning import DiffDB
def delete_all_xforms(cls, domain=None):
logger.debug("Deleting all Couch xforms for domain %s", domain)
cls._delete_all(XFormInstance.get_db(), all_known_formlike_doc_types(), domain)
FormProcessorTestUtils.delete_all_sql_forms(domain)
from couchforms.models import XFormInstance, doc_types as form_doc_types, all_known_formlike_doc_types
from dimagi.utils.couch.database import iter_docs
from dimagi.utils.couch.undo import DELETED_SUFFIX
from pillowtop.reindexer.change_providers.couch import CouchDomainDocTypeChangeProvider
from gevent.pool import Pool
from gevent import sleep
import six
import logging
from collections import defaultdict, deque
from corehq.util import cache_utils
_logger = logging.getLogger('main_couch_sql_datamigration')
CASE_DOC_TYPES = ['CommCareCase', 'CommCareCase-Deleted', ]
UNPROCESSED_DOC_TYPES = list(all_known_formlike_doc_types() - {'XFormInstance'})
def do_couch_to_sql_migration(domain, with_progress=True, debug=False):
set_local_domain_sql_backend_override(domain)
CouchSqlDomainMigrator(domain, with_progress=with_progress, debug=debug).migrate()
class CouchSqlDomainMigrator(object):
def __init__(self, domain, with_progress=True, debug=False):
from corehq.apps.tzmigration.planning import DiffDB
self._assert_no_migration_restrictions(domain)
self.with_progress = with_progress
self.debug = debug
self.domain = domain
db_filepath = get_diff_db_filepath(domain)
def _get_couch_forms_by_doc_type(domain):
return _get_couch_doc_counts(CommCareCase.get_db(), domain, all_known_formlike_doc_types())
