def _do_migration_and_assert_flags(self, domain):
self.assertFalse(should_use_sql_backend(domain))
call_command('migrate_domain_from_couch_to_sql',
domain,
MIGRATE=True,
no_input=True)
self.assertTrue(should_use_sql_backend(domain))
def test_duplicate_form_published(self):
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id)
orig_form = submit_form_locally(form_xml, domain=self.domain)[1]
self.assertEqual(form_id, orig_form.form_id)
self.assertEqual(1, len(self.form_accessors.get_all_form_ids_in_domain()))
with process_kafka_changes(self.form_pillow):
with process_couch_changes('DefaultChangeFeedPillow'):
# post an exact duplicate
dupe_form = submit_form_locally(form_xml, domain=self.domain)[1]
self.assertTrue(dupe_form.is_duplicate)
self.assertNotEqual(form_id, dupe_form.form_id)
if should_use_sql_backend(self.domain):
self.assertEqual(form_id, dupe_form.orig_id)
# make sure changes made it to kafka
dupe_form_meta = self.processor.changes_seen[0].metadata
self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id)
self.assertEqual(dupe_form.domain, dupe_form.domain)
if should_use_sql_backend(self.domain):
# sql domains also republish the original form to ensure that if the server crashed
# in the processing of the form the first time that it is still sent to kafka
orig_form_meta = self.processor.changes_seen[1].metadata
self.assertEqual(orig_form.form_id, orig_form_meta.document_id)
self.assertEqual(self.domain, orig_form_meta.domain)
self.assertEqual(dupe_form.domain, dupe_form.domain)
def test_duplicate_form_published(self):
form_id = uuid.uuid4().hex
form_xml = get_simple_form_xml(form_id)
orig_form = submit_form_locally(form_xml, domain=self.domain).xform
self.assertEqual(form_id, orig_form.form_id)
self.assertEqual(1,
len(self.form_accessors.get_all_form_ids_in_domain()))
with process_pillow_changes(self.form_pillow):
with process_pillow_changes('DefaultChangeFeedPillow'):
# post an exact duplicate
dupe_form = submit_form_locally(form_xml,
domain=self.domain).xform
self.assertTrue(dupe_form.is_duplicate)
self.assertNotEqual(form_id, dupe_form.form_id)
if should_use_sql_backend(self.domain):
self.assertEqual(form_id, dupe_form.orig_id)
# make sure changes made it to kafka
dupe_form_meta = self.processor.changes_seen[0].metadata
self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id)
self.assertEqual(dupe_form.domain, dupe_form.domain)
if should_use_sql_backend(self.domain):
# sql domains also republish the original form to ensure that if the server crashed
# in the processing of the form the first time that it is still sent to kafka
orig_form_meta = self.processor.changes_seen[1].metadata
self.assertEqual(orig_form.form_id, orig_form_meta.document_id)
self.assertEqual(self.domain, orig_form_meta.domain)
self.assertEqual(dupe_form.domain, dupe_form.domain)
def commit_migration(domain_name):
domain = Domain.get_by_name(domain_name, strict=True)
domain.use_sql_backend = True
domain.save()
clear_local_domain_sql_backend_override(domain_name)
if not should_use_sql_backend(domain_name):
Domain.get_by_name.clear(Domain, domain_name)
assert should_use_sql_backend(domain_name)
def commit_migration(domain_name):
domain = Domain.get_by_name(domain_name, strict=True)
domain.use_sql_backend = True
domain.save()
clear_local_domain_sql_backend_override(domain_name)
if not should_use_sql_backend(domain_name):
Domain.get_by_name.clear(Domain, domain_name)
assert should_use_sql_backend(domain_name)
datadog_counter("commcare.couch_sql_migration.total_committed")
_logger.info("committed migration for {}".format(domain))
def commit_migration(domain_name):
domain_obj = Domain.get_by_name(domain_name, strict=True)
domain_obj.use_sql_backend = True
domain_obj.save()
clear_local_domain_sql_backend_override(domain_name)
if not should_use_sql_backend(domain_name):
Domain.get_by_name.clear(Domain, domain_name)
assert should_use_sql_backend(domain_name)
datadog_counter("commcare.couch_sql_migration.total_committed")
_logger.info("committed migration for {}".format(domain_name))
def commit_migration(domain_name):
domain_obj = Domain.get_by_name(domain_name, strict=True)
domain_obj.use_sql_backend = True
domain_obj.save()
clear_local_domain_sql_backend_override(domain_name)
if not should_use_sql_backend(domain_name):
Domain.get_by_name.clear(Domain, domain_name)
assert should_use_sql_backend(domain_name), \
"could not set use_sql_backend for domain %s (try again)" % domain_name
datadog_counter("commcare.couch_sql_migration.total_committed")
log.info("committed migration for {}".format(domain_name))
def setUp(self):
super(TestHelperFunctions, self).setUp()
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.domain_name = uuid.uuid4().hex
self.domain = create_domain(self.domain_name)
self.assertFalse(should_use_sql_backend(self.domain_name))
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return
set_couch_sql_migration_started(domain)
with SignalHandlerContext([signal.SIGTERM, signal.SIGINT],
_get_sigterm_handler(domain)):
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
else:
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green(
"Domain migrated: {}".format(domain)))
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write("{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write("Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table(['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'], [
(doc_type,) + stat for doc_type, stat in stats.items()
])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def handle(self, domain, **options):
if not options['noinput']:
confirm = input("""
Are you sure you want to hard delete all forms and cases in domain "{}"?
This operation is PERMANENT.
Type the domain's name again to continue, or anything else to cancel:
""".format(domain))
if confirm != domain:
print("\n\t\tDomain deletion cancelled.")
return
assert should_use_sql_backend(domain)
logger.info('Hard deleting forms...')
deleted_sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(
domain)
for form_id_chunk in chunked(
with_progress_bar(deleted_sql_form_ids,
stream=silence_during_tests()), 500):
FormAccessorSQL.hard_delete_forms(domain,
list(form_id_chunk),
delete_attachments=True)
logger.info('Hard deleting cases...')
deleted_sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(
domain)
for case_id_chunk in chunked(
with_progress_bar(deleted_sql_case_ids,
stream=silence_during_tests()), 500):
CaseAccessorSQL.hard_delete_cases(domain, list(case_id_chunk))
logger.info('Done.')
def test_dry_run(self):
self.assertFalse(should_use_sql_backend(self.domain_name))
call_command(
'migrate_domain_from_couch_to_sql',
self.domain_name,
MIGRATE=True,
no_input=True,
dry_run=True
)
clear_local_domain_sql_backend_override(self.domain_name)
with self.assertRaises(CommandError):
call_command('migrate_domain_from_couch_to_sql', self.domain_name, COMMIT=True, no_input=True)
self.assertFalse(Domain.get_by_name(self.domain_name).use_sql_backend)
xml = """<?xml version="1.0" ?>
<n0:registration xmlns:n0="http://openrosa.org/user/registration">
<username>W4</username>
<password>2</password>
<uuid>P8DU7OLHVLZXU21JR10H3W8J2</uuid>
<date>2013-11-19</date>
<registering_phone_id>8H1N48EFPF6PA4UOO8YGZ2KFZ</registering_phone_id>
<user_data>
<data key="user_type">standard</data>
</user_data>
</n0:registration>
"""
submit_form_locally(xml, self.domain_name)
couch_form_ids = self._get_form_ids()
self.assertEqual(1, len(couch_form_ids))
call_command('migrate_domain_from_couch_to_sql', self.domain_name, blow_away=True, no_input=True)
self.assertFalse(Domain.get_by_name(self.domain_name).use_sql_backend)
def handle(self, domain, action, **options):
if should_use_sql_backend(domain):
raise CommandError(
'It looks like {} has already been migrated.'.format(domain))
for opt in [
"no_input",
"verbose",
"state_dir",
"live_migrate",
"diff_process",
"rebuild_state",
]:
setattr(self, opt, options[opt])
if self.no_input and not settings.UNIT_TESTING:
raise CommandError('--no-input only allowed for unit testing')
if action != MIGRATE and self.live_migrate:
raise CommandError("--live only allowed with `MIGRATE`")
if action != MIGRATE and self.rebuild_state:
raise CommandError("--rebuild-state only allowed with `MIGRATE`")
if action != STATS and self.verbose:
raise CommandError("--verbose only allowed for `stats`")
assert Domain.get_by_name(domain), f'Unknown domain "{domain}"'
slug = f"{action.lower()}-{domain}"
setup_logging(self.state_dir, slug, options['debug'])
getattr(self, "do_" + action)(domain)
def migrate_domain(self, domain, state_dir):
if should_use_sql_backend(domain):
log.info("{} already on the SQL backend\n".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
log.error("{} migration is in progress\n".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain, self.live_migrate)
try:
do_couch_to_sql_migration(
domain,
state_dir,
with_progress=self.live_migrate,
live_migrate=self.live_migrate,
)
except MigrationRestricted as err:
log.error("migration restricted: %s", err)
set_couch_sql_migration_not_started(domain)
return False, str(err)
has_diffs = self.check_diffs(domain, state_dir)
if self.live_migrate:
return True, None
if has_diffs:
self.abort(domain, state_dir)
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
log.info("Domain migrated: {}\n".format(domain))
return True, None
def handle(self, user, **options):
try:
self.user = CouchUser.get_by_username(user)
if not self.user:
self.user = CouchUser.get(user)
except ResourceNotFound:
print("Could not find user {}".format(user))
return
if not isinstance(self.user, CommCareUser):
print ("Sorry, the user you specify has to be a mobile worker. "
"This changed when delete_cases was refactored to use "
"cases_by_owner/view instead of case/by_owner. "
"The new view needs an explicit domain, "
"and I didn't implement that for WebUsers who can belong "
"to multiple domains, but usually do not own cases.")
exit(1)
self.domain = self.user.domain
if should_use_sql_backend(self.domain):
raise CommandError('This command only works for couch-based domains.')
if not options.get('no_prompt'):
msg = "Delete all cases owned by {}? (y/n)\n".format(
self.user.username,
)
if not input(msg) == 'y':
print("cancelling")
return
self.delete_all()
print("Cases successfully deleted, you monster!")
def migrate_domain(self, domain, state_dir):
if should_use_sql_backend(domain):
log.info("{} already on the SQL backend\n".format(domain))
return
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
log.error("{} migration is in progress\n".format(domain))
raise Incomplete("in progress")
set_couch_sql_migration_started(domain, self.live_migrate)
try:
do_couch_to_sql_migration(
domain,
state_dir,
with_progress=self.live_migrate,
live_migrate=self.live_migrate,
)
except MigrationRestricted as err:
log.error("migration restricted: %s", err)
set_couch_sql_migration_not_started(domain)
raise Incomplete(str(err))
if self.live_migrate:
self.finish_live_migration_if_possible(domain, state_dir)
elif self.has_diffs(domain, state_dir):
self.abort(domain, state_dir)
raise Incomplete("has diffs")
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
log.info(f"Domain migrated: {domain}\n")
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
log.error("{} already on the SQL backend\n".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
log.error("{} migration is already in progress\n".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
lines = ["Migration has diffs: {}".format(domain)]
class stream:
write = lines.append
writer = SimpleTableWriter(stream, TableRowFormatter([30, 10, 10, 10, 10]))
writer.write_table(
['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'],
[(doc_type,) + stat for doc_type, stat in stats.items()],
)
log.error("\n".join(lines))
self.abort(domain)
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
log.info("Domain migrated: {}\n".format(domain))
return True, None
def handle(self, action, domain, **options):
if should_use_sql_backend(domain):
raise CommandError(f'It looks like {domain} has already been migrated.')
for opt in [
"no_input",
"debug",
"state_path",
"select",
"stop",
"changes",
"batch_size",
"reset",
]:
setattr(self, opt, options[opt])
if self.no_input and not settings.UNIT_TESTING:
raise CommandError('--no-input only allowed for unit testing')
if self.changes and action != SHOW:
raise CommandError('--changes only allowed with "show" action')
if self.reset:
if action == SHOW:
raise CommandError(f'invalid action for --reset: {action}')
self.do_reset(action, domain)
return
if action != SHOW:
assert Domain.get_by_name(domain), f'Unknown domain "{domain}"'
do_action = getattr(self, "do_" + action)
msg = do_action(domain)
if msg:
sys.exit(msg)
def handle(self, user, **options):
try:
self.user = CouchUser.get_by_username(user)
if not self.user:
self.user = CouchUser.get(user)
except ResourceNotFound:
print("Could not find user {}".format(user))
return
if not isinstance(self.user, CommCareUser):
print("Sorry, the user you specify has to be a mobile worker. "
"This changed when delete_cases was refactored to use "
"cases_by_owner/view instead of case/by_owner. "
"The new view needs an explicit domain, "
"and I didn't implement that for WebUsers who can belong "
"to multiple domains, but usually do not own cases.")
exit(1)
self.domain = self.user.domain
if should_use_sql_backend(self.domain):
raise CommandError(
'This command only works for couch-based domains.')
if not options.get('no_prompt'):
msg = "Delete all cases owned by {}? (y/n)\n".format(
self.user.username, )
if not raw_input(msg) == 'y':
print("cancelling")
return
self.delete_all()
print("Cases successfully deleted, you monster!")
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write(
"{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def migrate_domain(self, domain, state_dir):
if should_use_sql_backend(domain):
log.info("{} already on the SQL backend\n".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
log.error("{} migration is in progress\n".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
try:
do_couch_to_sql_migration(domain, state_dir, with_progress=False)
except MigrationRestricted as err:
log.error("migration restricted: %s", err)
set_couch_sql_migration_not_started(domain)
return False, str(err)
stats = get_diff_stats(domain, state_dir, self.strict)
if stats:
header = "Migration has diffs: {}".format(domain)
log.error(format_diff_stats(stats, header))
self.abort(domain, state_dir)
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
log.info("Domain migrated: {}\n".format(domain))
return True, None
def handle(self, *args, **options):
if len(args) == 1:
domain = args[0]
since = None
elif len(args) == 2:
domain = args[0]
since = string_to_datetime(args[1])
else:
raise CommandError('Usage: %s\n%s' % (self.args, self.help))
if should_use_sql_backend(domain):
raise CommandError('This command only works for couch-based domains.')
succeeded = []
failed = []
error_messages = defaultdict(lambda: 0)
for form in iter_problem_forms(domain, since):
print "%s\t%s\t%s\t%s\t%s" % (form._id, form.received_on,
form.xmlns,
form.get_data('form/meta/username'),
form.problem.strip())
if not options["dryrun"]:
try:
reprocess_form_cases(form)
except Exception, e:
failed.append(form._id)
error_messages[str(e)] += 1
else:
succeeded.append(form._id)
def test_dry_run(self):
self.assertFalse(should_use_sql_backend(self.domain_name))
call_command(
'migrate_domain_from_couch_to_sql',
self.domain_name,
MIGRATE=True,
no_input=True,
dry_run=True
)
clear_local_domain_sql_backend_override(self.domain_name)
with self.assertRaises(CommandError):
call_command('migrate_domain_from_couch_to_sql', self.domain_name, COMMIT=True, no_input=True)
self.assertFalse(Domain.get_by_name(self.domain_name).use_sql_backend)
xml = """<?xml version="1.0" ?>
<n0:registration xmlns:n0="http://openrosa.org/user/registration">
<username>W4</username>
<password>2</password>
<uuid>P8DU7OLHVLZXU21JR10H3W8J2</uuid>
<date>2013-11-19</date>
<registering_phone_id>8H1N48EFPF6PA4UOO8YGZ2KFZ</registering_phone_id>
<user_data>
<data key="user_type">standard</data>
</user_data>
</n0:registration>
"""
submit_form_locally(xml, self.domain_name)
couch_form_ids = self._get_form_ids()
self.assertEqual(1, len(couch_form_ids))
call_command('migrate_domain_from_couch_to_sql', self.domain_name, blow_away=True, no_input=True)
self.assertFalse(Domain.get_by_name(self.domain_name).use_sql_backend)
def property_changed_in_action(domain, case_transaction, case_id, case_property_name):
from casexml.apps.case.xform import get_case_updates
PropertyChangedInfo = namedtuple("PropertyChangedInfo", 'transaction new_value modified_on')
include_create_fields = case_property_name in ['owner_id', 'name', 'external_id']
if not should_use_sql_backend(domain):
# couch domains return 2 transactions for case properties created in a create form
if case_transaction.is_case_create and not include_create_fields:
return False
case_updates = get_case_updates(case_transaction.form)
actions = []
for update in case_updates:
if update.id == case_id:
actions.append((update.modified_on_str, update.get_update_action(), case_transaction))
if include_create_fields and case_transaction.is_case_create:
actions.append((update.modified_on_str, update.get_create_action(), case_transaction))
for (modified_on, action, case_transaction) in actions:
if action:
property_changed = action.dynamic_properties.get(case_property_name)
if include_create_fields and not property_changed:
property_changed = getattr(action, case_property_name, None)
if property_changed is not None:
return PropertyChangedInfo(case_transaction, property_changed, modified_on)
return False
def _get_form_ids(self, domain):
if should_use_sql_backend(domain):
problem_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(
domain, 'XFormError')
else:
problem_ids = get_form_ids_by_type(domain, 'XFormError')
return problem_ids
def handle(self, *args, **options):
if len(args) == 1:
domain = args[0]
since = None
elif len(args) == 2:
domain = args[0]
since = string_to_datetime(args[1])
else:
raise CommandError('Usage: %s\n%s' % (self.args, self.help))
if should_use_sql_backend(domain):
raise CommandError(
'This command only works for couch-based domains.')
succeeded = []
failed = []
error_messages = defaultdict(lambda: 0)
for form in iter_problem_forms(domain, since):
print "%s\t%s\t%s\t%s\t%s" % (
form._id, form.received_on, form.xmlns,
form.get_data('form/meta/username'), form.problem.strip())
if not options["dryrun"]:
try:
reprocess_form_cases(form)
except Exception, e:
failed.append(form._id)
error_messages[str(e)] += 1
else:
succeeded.append(form._id)
def resave_form(domain, form):
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.change_publishers import publish_form_saved
from couchforms.models import XFormInstance
if should_use_sql_backend(domain):
publish_form_saved(form)
else:
XFormInstance.get_db().save_doc(form.to_json())
def resave_form(domain, form):
from corehq.form_processor.utils import should_use_sql_backend
from corehq.form_processor.change_publishers import publish_form_saved
from couchforms.models import XFormInstance
if should_use_sql_backend(domain):
publish_form_saved(form)
else:
XFormInstance.get_db().save_doc(form.to_json())
def print_migration(self, item):
started = item.started_on
migrated = should_use_sql_backend(item.domain)
print(" {}{}{}".format(
item.domain,
started.strftime(" (%Y-%m-%d)") if started else "",
" - already migrated? (bad state)" if migrated else "",
))
def _assert_no_migration_restrictions(self, domain_name):
assert should_use_sql_backend(domain_name)
assert not COUCH_SQL_MIGRATION_BLACKLIST.enabled(
domain_name, NAMESPACE_DOMAIN)
assert not any(
custom_report_domain == domain_name
for custom_report_domain in settings.DOMAIN_MODULE_MAP.keys())
assert not REMINDERS_MIGRATION_IN_PROGRESS.enabled(domain_name)
def resave_case(domain, case, send_post_save_signal=True):
from corehq.form_processor.change_publishers import publish_case_saved
if should_use_sql_backend(domain):
publish_case_saved(case, send_post_save_signal)
else:
if send_post_save_signal:
case.save()
else:
CommCareCase.get_db().save_doc(case._doc) # don't just call save to avoid signals
def get_case_ids_for_messaging_rule(domain, case_type):
if not should_use_sql_backend(domain):
return CaseAccessors(domain).get_case_ids_in_domain(case_type)
else:
return run_query_across_partitioned_databases(
CommCareCaseSQL,
Q(domain=domain, type=case_type, deleted=False),
values=['case_id']
)
def setUp(self):
super(BaseMigrationTestCase, self).setUp()
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.domain_name = uuid.uuid4().hex
self.domain = create_domain(self.domain_name)
# all new domains are set complete when they are created
DomainMigrationProgress.objects.filter(domain=self.domain_name).delete()
self.assertFalse(should_use_sql_backend(self.domain_name))
def handle(self, *args, **options):
if len(args) < 2:
raise CommandError('Usage is copy_case, %s' % self.args)
source_couch = CouchConfig(args[0])
case_id = args[1]
doc_ids = [case_id]
domain = args[2] if len(args) > 2 else None
if should_use_sql_backend(domain):
raise CommandError('This command only works for couch-based domains.')
def _migrate_case(case_id):
print 'getting case %s' % case_id
case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id))
original_domain = case.domain
if domain is not None:
case.domain = domain
case.save(force_update=True)
return case, original_domain
case, orig_domain = _migrate_case(case_id)
print 'copying %s parent cases' % len(case.indices)
for index in case.indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
# hack, set the domain back to make sure we get the reverse indices correctly
case.domain = orig_domain
with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)):
child_indices = get_reverse_indices(case)
print 'copying %s child cases' % len(child_indices)
for index in child_indices:
_migrate_case(index.referenced_id)
doc_ids.append(index.referenced_id)
print 'copying %s xforms' % len(case.xform_ids)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
doc.pop('external_blobs', None)
return XFormInstance.wrap(doc)
xforms = source_couch.get_db_for_class(XFormInstance).all_docs(
keys=case.xform_ids,
include_docs=True,
wrapper=form_wrapper,
).all()
for form in xforms:
if domain is not None:
form.domain = domain
form.save(force_update=True)
print 'saved %s' % form._id
doc_ids.append(form._id)
if options['postgres_db']:
copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
def setUp(self):
super(BaseMigrationTestCase, self).setUp()
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.domain_name = uuid.uuid4().hex
self.domain = create_domain(self.domain_name)
# all new domains are set complete when they are created
DomainMigrationProgress.objects.filter(domain=self.domain_name).delete()
self.assertFalse(should_use_sql_backend(self.domain_name))
def get_case_ids_for_messaging_rule(domain, case_type):
if not should_use_sql_backend(domain):
return CaseAccessors(domain).get_case_ids_in_domain(case_type)
else:
return run_query_across_partitioned_databases(
CommCareCaseSQL,
Q(domain=domain, type=case_type, deleted=False),
values=['case_id']
)
def handle(self, domain, **options):
if should_use_sql_backend(domain):
raise CommandError('It looks like {} has already been migrated.'.format(domain))
self.no_input = options.pop('no_input', False)
self.debug = options.pop('debug', False)
self.dry_run = options.pop('dry_run', False)
if self.no_input and not settings.UNIT_TESTING:
raise CommandError('no-input only allowed for unit testing')
setup_logging(options['log_dir'])
if options['MIGRATE']:
self.require_only_option('MIGRATE', options)
if options.get('run_timestamp'):
if not couch_sql_migration_in_progress(domain):
raise CommandError("Migration must be in progress if run_timestamp is passed in")
else:
set_couch_sql_migration_started(domain, self.dry_run)
do_couch_to_sql_migration(
domain,
with_progress=not self.no_input,
debug=self.debug,
run_timestamp=options.get('run_timestamp'))
has_diffs = self.print_stats(domain, short=True, diffs_only=True)
if has_diffs:
print("\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n")
if options['blow_away']:
self.require_only_option('blow_away', options)
if not self.no_input:
_confirm(
"This will delete all SQL forms and cases for the domain {}. "
"Are you sure you want to continue?".format(domain)
)
set_couch_sql_migration_not_started(domain)
blow_away_migration(domain)
if options['stats_short'] or options['stats_long']:
self.print_stats(domain, short=options['stats_short'])
if options['show_diffs']:
self.show_diffs(domain)
if options['COMMIT']:
self.require_only_option('COMMIT', options)
if not couch_sql_migration_in_progress(domain, include_dry_runs=False):
raise CommandError("cannot commit a migration that is not in state in_progress")
if not self.no_input:
_confirm(
"This will convert the domain to use the SQL backend and"
"allow new form submissions to be processed. "
"Are you sure you want to do this for domain '{}'?".format(domain)
)
set_couch_sql_migration_complete(domain)
def handle(self, domain, action, **options):
if action != STATS and should_use_sql_backend(domain):
raise CommandError(
'It looks like {} has already been migrated.'.format(domain))
if options["patch"]:
if options["forms"]:
raise CommandError(
"--patch and --forms=... are mutually exclusive")
if options["case_diff"] != "after":
raise CommandError(
"--patch and --case-diff=... are mutually exclusive")
options["forms"] = "missing"
options["case_diff"] = "patch"
for opt in [
"no_input",
"verbose",
"state_dir",
"live_migrate",
"finish",
"case_diff",
"rebuild_state",
"stop_on_error",
"forms",
"rewind",
"missing_docs",
]:
setattr(self, opt, options[opt])
if self.no_input and not settings.UNIT_TESTING:
raise CommandError('--no-input only allowed for unit testing')
if action != MIGRATE and self.live_migrate:
raise CommandError(f"{action} --live not allowed")
if action != MIGRATE and self.finish:
raise CommandError(f"{action} --finish not allowed")
if action != MIGRATE and self.rebuild_state:
raise CommandError("--rebuild-state only allowed with `MIGRATE`")
if action != MIGRATE and self.forms:
raise CommandError("--forms only allowed with `MIGRATE`")
if action != MIGRATE and self.stop_on_error:
raise CommandError("--stop-on-error only allowed with `MIGRATE`")
if action != STATS and self.verbose:
raise CommandError("--verbose only allowed for `stats`")
if action not in [MIGRATE, STATS] and self.missing_docs != CACHED:
raise CommandError(f"{action} --missing-docs not allowed")
if action != REWIND and self.rewind:
raise CommandError("--to=... only allowed for `rewind`")
if self.case_diff == "patch" and self.forms not in [None, "missing"]:
raise CommandError(
f"not supported: --case-diff=patch --forms={self.forms}")
assert Domain.get_by_name(domain), f'Unknown domain "{domain}"'
slug = f"{action.lower()}-{domain}"
setup_logging(self.state_dir, slug, options['debug'])
getattr(self, "do_" + action)(domain)
def _check_for_migration_restrictions(self, domain_name):
msgs = []
if not should_use_sql_backend(domain_name):
msgs.append("does not have SQL backend enabled")
if COUCH_SQL_MIGRATION_BLACKLIST.enabled(domain_name, NAMESPACE_DOMAIN):
msgs.append("is blacklisted")
if domain_name in settings.DOMAIN_MODULE_MAP:
msgs.append("has custom reports")
if msgs:
raise MigrationRestricted("{}: {}".format(domain_name, "; ".join(msgs)))
def get_document_store(domain, doc_type):
use_sql = should_use_sql_backend(domain)
if use_sql and doc_type == 'XFormInstance':
return ReadonlyFormDocumentStore(domain)
elif use_sql and doc_type == 'CommCareCase':
return ReadonlyCaseDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type)
def __init__(self, domain, with_progress=True, debug=False):
from corehq.apps.tzmigration.planning import DiffDB
assert should_use_sql_backend(domain)
self.with_progress = with_progress
self.debug = debug
self.domain = domain
db_filepath = get_diff_db_filepath(domain)
self.diff_db = DiffDB.init(db_filepath)
self.errors_with_normal_doc_type = []
def _check_for_migration_restrictions(self, domain_name):
msgs = []
if not should_use_sql_backend(domain_name):
msgs.append("does not have SQL backend enabled")
if COUCH_SQL_MIGRATION_BLACKLIST.enabled(domain_name,
NAMESPACE_DOMAIN):
msgs.append("is blacklisted")
if domain_name in settings.DOMAIN_MODULE_MAP:
msgs.append("has custom reports")
if msgs:
raise MigrationRestricted("{}: {}".format(domain_name,
"; ".join(msgs)))
def __init__(self, domain, with_progress=True, debug=False):
from corehq.apps.tzmigration.planning import DiffDB
assert should_use_sql_backend(domain)
self.with_progress = with_progress
self.debug = debug
self.domain = domain
db_filepath = get_diff_db_filepath(domain)
self.diff_db = DiffDB.init(db_filepath)
self.errors_with_normal_doc_type = []
self.forms_that_touch_cases_without_actions = set()
def handle(self, domain, reason, **options):
if should_use_sql_backend(domain):
raise CommandError('This command only works for couch-based domains.')
ids = get_case_ids_in_domain(domain)
for count, case_id in enumerate(ids):
try:
rebuild_case_from_forms(domain, case_id, RebuildWithReason(reason=reason))
if count % 100 == 0:
print('rebuilt %s/%s cases' % (count, len(ids)))
except Exception as e:
logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
def get_document_store(domain, doc_type):
use_sql = should_use_sql_backend(domain)
if use_sql and doc_type == 'XFormInstance':
return ReadonlyFormDocumentStore(domain)
elif use_sql and doc_type == 'CommCareCase':
return ReadonlyCaseDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(
couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type
)
def setUp(self):
super(BaseMigrationTestCase, self).setUp()
with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"):
config = settings.S3_BLOB_DB_SETTINGS
self.s3db = TemporaryS3BlobDB(config)
assert get_blob_db() is self.s3db, (get_blob_db(), self.s3db)
FormProcessorTestUtils.delete_all_cases_forms_ledgers()
self.domain_name = uuid.uuid4().hex
self.domain = create_domain(self.domain_name)
# all new domains are set complete when they are created
DomainMigrationProgress.objects.filter(domain=self.domain_name).delete()
self.assertFalse(should_use_sql_backend(self.domain_name))
def handle(self, domain, doc_type, **options):
csv = options.get('csv')
startdate = options.get('start')
enddate = options.get('end')
form_doc_types = doc_types()
if startdate or enddate:
if doc_type in CASE_DOC_TYPES or doc_type in form_doc_types:
if not should_use_sql_backend(domain):
raise CommandError("Date filtering not supported for Couch domains")
if startdate and enddate and enddate <= startdate:
raise CommandError("enddate must be after startdate")
handlers = {
'CommCareCase': compare_cases,
'CommCareCase-Deleted': compare_cases,
'CommCareUser': _compare_users,
'CommCareUser-Deleted': _compare_users,
'WebUser': _compare_users,
}
handlers.update({doc_type: compare_xforms for doc_type in form_doc_types})
try:
primary_count, es_count, primary_only, es_only = handlers[doc_type](domain, doc_type, startdate, enddate)
except KeyError:
raise CommandError('Unsupported doc type. Use on of: {}'.format(', '.join(handlers)))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([50, 50])
date_range_output = ''
if startdate or enddate:
end = (enddate or datetime.utcnow().date()).strftime(DATE_FORMAT)
start = startdate.strftime(DATE_FORMAT)
date_range_output = ' (Between {} and {})'.format(start, end)
print("\nDoc ID analysis for {}{}\n".format(doc_type, date_range_output))
print("Primary Count: {}".format(primary_count))
print("ES Count: {}\n".format(es_count))
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(
['Only in Primary', 'Only in ES'],
zip_longest(primary_only, es_only, fillvalue='')
)
def _blow_away_migration(domain):
assert not should_use_sql_backend(domain)
delete_diff_db(domain)
for doc_type in doc_types():
sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
def sync_cases(self, domain):
db_aliases = get_db_aliases_for_partitioned_query()
db_aliases.sort()
if should_use_sql_backend(domain):
case_accessor = CaseReindexAccessor(domain)
case_ids = (case.case_id for case in iter_all_rows(case_accessor))
else:
changes = _get_case_iterator(domain).iter_all_changes()
case_ids = (case.id for case in changes)
next_event = time.time() + 10
for i, case_id in enumerate(case_ids):
sync_case_for_messaging.delay(domain, case_id)
if time.time() > next_event:
print("Queued %d cases for domain %s" % (i + 1, domain))
next_event = time.time() + 10
def handle(self, domain, csv_file, **options):
self.domain = domain
if not should_use_sql_backend(domain):
print("This domain doesn't use SQL backend, exiting!")
return
current_date = self.first_form_received_on()
if not current_date:
print("No submissions in this domain yet, exiting!")
return
with open(csv_file, "w", encoding='utf-8') as csv_file:
field_names = ('date', 'doc_type', 'in_sql', 'in_es', 'diff')
csv_writer = csv.DictWriter(csv_file, field_names, extrasaction='ignore')
csv_writer.writeheader()
while current_date <= datetime.today():
cases_in_sql = self._get_sql_cases_modified_on_date(current_date)
cases_in_es = self._get_es_cases_modified_on_date(current_date)
properties = {
"date": current_date,
"doc_type": "CommCareCase",
"in_sql": cases_in_sql,
"in_es": cases_in_es,
"diff": cases_in_sql - cases_in_es,
}
csv_writer.writerow(properties)
print(properties)
forms_in_sql = self._get_sql_forms_received_on_date(current_date)
forms_in_es = self._get_es_forms_received_on_date(current_date)
properties = {
"date": current_date,
"doc_type": "XFormInstance",
"in_sql": forms_in_sql,
"in_es": forms_in_es,
"diff": forms_in_sql - forms_in_es
}
csv_writer.writerow(properties)
print(properties)
current_date += relativedelta(months=1)
def handle(self, domain, **options):
if should_use_sql_backend(domain):
raise CommandError('This command only works for couch-based domains.')
filepath = get_planning_db_filepath(domain)
self.stdout.write('Using file {}\n'.format(filepath))
if options['BEGIN']:
self.require_only_option('BEGIN', options)
set_tz_migration_started(domain)
if options['ABORT']:
self.require_only_option('ABORT', options)
set_tz_migration_not_started(domain)
if options['blow_away']:
delete_planning_db(domain)
self.stdout.write('Removed file {}\n'.format(filepath))
if options['prepare']:
self.planning_db = prepare_planning_db(domain)
self.stdout.write('Created and loaded file {}\n'.format(filepath))
else:
self.planning_db = get_planning_db(domain)
if options['COMMIT']:
self.require_only_option('COMMIT', options)
assert get_tz_migration_status(domain, strict=True) == MigrationStatus.IN_PROGRESS
commit_plan(domain, self.planning_db)
set_tz_migration_complete(domain)
if options['prepare_case_json']:
prepare_case_json(self.planning_db)
if options['stats']:
self.valiate_forms_and_cases(domain)
if options['show_diffs']:
self.show_diffs()
if options['play']:
from corehq.apps.tzmigration.planning import *
session = self.planning_db.Session() # noqa
try:
import ipdb as pdb
except ImportError:
import pdb
pdb.set_trace()
def handle_label(self, domain, **options):
if should_use_sql_backend(domain):
raise CommandError(u'It looks like {} has already been migrated.'.format(domain))
self.no_input = options.pop('no_input', False)
self.debug = options.pop('debug', False)
if self.no_input and not settings.UNIT_TESTING:
raise CommandError('no-input only allowed for unit testing')
if options['MIGRATE']:
self.require_only_option('MIGRATE', options)
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=not self.no_input, debug=self.debug)
has_diffs = self.print_stats(domain, short=True, diffs_only=True)
if has_diffs:
print "\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n"
if options['blow_away']:
self.require_only_option('blow_away', options)
if not self.no_input:
_confirm(
"This will delete all SQL forms and cases for the domain {}. "
"Are you sure you want to continue?".format(domain)
)
set_couch_sql_migration_not_started(domain)
_blow_away_migration(domain)
if options['stats_short'] or options['stats_long']:
self.print_stats(domain, short=options['stats_short'])
if options['show_diffs']:
self.show_diffs(domain)
if options['COMMIT']:
self.require_only_option('COMMIT', options)
assert couch_sql_migration_in_progress(domain)
if not self.no_input:
_confirm(
"This will allow convert the domain to use the SQL backend and"
"allow new form submissions to be processed. "
"Are you sure you want to do this for domain '{}'?".format(domain)
)
set_couch_sql_migration_complete(domain)
def filter_cases(request, domain, app_id, module_id, parent_id=None):
app = Application.get(app_id)
module = app.get_module(module_id)
auth_cookie = request.COOKIES.get('sessionid')
requires_parent_cases = string_to_boolean(request.GET.get('requires_parent_cases', 'false'))
xpath = EntriesHelper.get_filter_xpath(module)
instances = get_instances_for_module(app, module, additional_xpaths=[xpath])
extra_instances = [{'id': inst.id, 'src': inst.src} for inst in instances]
accessor = CaseAccessors(domain)
# touchforms doesn't like this to be escaped
xpath = HTMLParser.HTMLParser().unescape(xpath)
case_type = module.case_type
if xpath or should_use_sql_backend(domain):
# if we need to do a custom filter, send it to touchforms for processing
additional_filters = {
"properties/case_type": case_type,
"footprint": True
}
helper = BaseSessionDataHelper(domain, request.couch_user)
result = helper.filter_cases(xpath, additional_filters, DjangoAuth(auth_cookie),
extra_instances=extra_instances)
if result.get('status', None) == 'error':
code = result.get('code', 500)
message = result.get('message', _("Something went wrong filtering your cases."))
if code == 500:
notify_exception(None, message=message)
return json_response(message, status_code=code)
case_ids = result.get("cases", [])
else:
# otherwise just use our built in api with the defaults
case_ids = [res.id for res in get_filtered_cases(
domain,
status=CASE_STATUS_OPEN,
case_type=case_type,
user_id=request.couch_user._id,
footprint=True,
ids_only=True,
)]
cases = accessor.get_cases(case_ids)
if parent_id:
cases = filter(lambda c: c.parent and c.parent.case_id == parent_id, cases)
# refilter these because we might have accidentally included footprint cases
# in the results from touchforms. this is a little hacky but the easiest
# (quick) workaround. should be revisted when we optimize the case list.
cases = filter(lambda c: c.type == case_type, cases)
cases = [c.to_api_json(lite=True) for c in cases if c]
response = {'cases': cases}
if requires_parent_cases:
# Subtract already fetched cases from parent list
parent_ids = set(map(lambda c: c['indices']['parent']['case_id'], cases)) - \
set(map(lambda c: c['case_id'], cases))
parents = accessor.get_cases(list(parent_ids))
parents = [c.to_api_json(lite=True) for c in parents]
response.update({'parents': parents})
return json_response(response)
def REPORTS(project):
from corehq.apps.reports.standard.cases.basic import CaseListReport
report_set = None
if project.report_whitelist:
report_set = set(project.report_whitelist)
reports = []
reports.extend(_get_configurable_reports(project))
monitoring_reports = (
monitoring.WorkerActivityReport,
monitoring.DailyFormStatsReport,
monitoring.SubmissionsByFormReport,
monitoring.FormCompletionTimeReport,
monitoring.CaseActivityReport,
monitoring.FormCompletionVsSubmissionTrendsReport,
monitoring.WorkerActivityTimes,
ProjectHealthDashboard,
)
inspect_reports = [
inspect.SubmitHistory, CaseListReport, OdmExportReport,
]
if toggles.CASE_LIST_EXPLORER.enabled(project.name):
inspect_reports.append(CaseListExplorer)
deployments_reports = (
deployments.ApplicationStatusReport,
deployments.AggregateUserStatusReport,
receiverwrapper.SubmissionErrorReport,
phonelog.DeviceLogDetailsReport,
deployments.ApplicationErrorReport,
)
monitoring_reports = _filter_reports(report_set, monitoring_reports)
inspect_reports = _filter_reports(report_set, inspect_reports)
deployments_reports = _filter_reports(report_set, deployments_reports)
reports.extend([
(ugettext_lazy("Monitor Workers"), monitoring_reports),
(ugettext_lazy("Inspect Data"), inspect_reports),
(ugettext_lazy("Manage Deployments"), deployments_reports),
])
if project.commtrack_enabled:
supply_reports = (
commtrack.SimplifiedInventoryReport,
commtrack.InventoryReport,
commtrack.CurrentStockStatusReport,
commtrack.StockStatusMapReport,
)
if not should_use_sql_backend(project):
supply_reports = supply_reports + (
commtrack.ReportingRatesReport,
commtrack.ReportingStatusMapReport,
)
supply_reports = _filter_reports(report_set, supply_reports)
reports.insert(0, (ugettext_lazy("CommCare Supply"), supply_reports))
reports = list(_get_report_builder_reports(project)) + reports
from corehq.apps.accounting.utils import domain_has_privilege
messaging_reports = []
project_can_use_sms = domain_has_privilege(project.name, privileges.OUTBOUND_SMS)
if project_can_use_sms:
messaging_reports.extend([
sms.MessagesReport,
])
# always have these historical reports visible
messaging_reports.extend([
sms.MessagingEventsReport,
sms.MessageEventDetailReport,
sms.SurveyDetailReport,
sms.MessageLogReport,
sms.SMSOptOutReport,
ivr.CallReport,
ivr.ExpectedCallbackReport,
sms.PhoneNumberReport,
sms.ScheduleInstanceReport,
])
messaging_reports += getattr(Domain.get_module_by_name(project.name), 'MESSAGING_REPORTS', ())
messaging_reports = _filter_reports(report_set, messaging_reports)
messaging = (ugettext_lazy("Messaging"), messaging_reports)
reports.append(messaging)
reports.extend(_get_dynamic_reports(project))
return reports
def _assert_no_migration_restrictions(self, domain_name):
assert should_use_sql_backend(domain_name)
assert not COUCH_SQL_MIGRATION_BLACKLIST.enabled(domain_name, NAMESPACE_DOMAIN)
assert not any(custom_report_domain == domain_name
for custom_report_domain in settings.DOMAIN_MODULE_MAP.keys())
assert not REMINDERS_MIGRATION_IN_PROGRESS.enabled(domain_name)
def _process_form(request, domain, app_id, user_id, authenticated,
auth_cls=AuthContext):
metric_tags = [
'backend:sql' if should_use_sql_backend(domain) else 'backend:couch',
'domain:{}'.format(domain),
]
if should_ignore_submission(request):
# silently ignore submission if it meets ignore-criteria
response = openrosa_response.SUBMISSION_IGNORED_RESPONSE
_record_metrics(metric_tags, 'ignored', response)
return response
if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain):
response = openrosa_response.BLACKLISTED_RESPONSE
_record_metrics(metric_tags, 'blacklisted', response)
return response
with TimingContext() as timer:
try:
instance, attachments = couchforms.get_instance_and_attachment(request)
except MultimediaBug as e:
try:
instance = request.FILES[MAGIC_PROPERTY].read()
xform = convert_xform_to_json(instance)
meta = xform.get("meta", {})
except:
meta = {}
return _submission_error(
request, "Received a submission with POST.keys()",
MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags,
domain, app_id, user_id, authenticated, meta,
)
app_id, build_id = get_app_and_build_ids(domain, app_id)
submission_post = SubmissionPost(
instance=instance,
attachments=attachments,
domain=domain,
app_id=app_id,
build_id=build_id,
auth_context=auth_cls(
domain=domain,
user_id=user_id,
authenticated=authenticated,
),
location=couchforms.get_location(request),
received_on=couchforms.get_received_on(request),
date_header=couchforms.get_date_header(request),
path=couchforms.get_path(request),
submit_ip=couchforms.get_submit_ip(request),
last_sync_token=couchforms.get_last_sync_token(request),
openrosa_headers=couchforms.get_openrosa_headers(request),
)
try:
result = submission_post.run()
except XFormLockError as err:
return _submission_error(
request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT,
metric_tags, domain, app_id, user_id, authenticated, status=423,
notify=False,
)
response = result.response
if response.status_code == 400:
logging.error(
'Status code 400 for a form submission. '
'Response is: \n{0}\n'
)
_record_metrics(metric_tags, result.submission_type, response, result, timer)
return response
def REPORTS(project):
from corehq.apps.reports.standard.cases.basic import CaseListReport
from corehq.apps.reports.standard.cases.careplan import make_careplan_reports
reports = []
reports.extend(_get_configurable_reports(project))
reports.extend([
(ugettext_lazy("Monitor Workers"), (
monitoring.WorkerActivityReport,
monitoring.DailyFormStatsReport,
monitoring.SubmissionsByFormReport,
monitoring.FormCompletionTimeReport,
monitoring.CaseActivityReport,
monitoring.FormCompletionVsSubmissionTrendsReport,
monitoring.WorkerActivityTimes,
ProjectHealthDashboard,
)),
(ugettext_lazy("Inspect Data"), (
inspect.SubmitHistory, CaseListReport, OdmExportReport,
)),
(ugettext_lazy("Manage Deployments"), (
deployments.ApplicationStatusReport,
receiverwrapper.SubmissionErrorReport,
phonelog.DeviceLogDetailsReport,
deployments.SyncHistoryReport,
deployments.ApplicationErrorReport,
)),
])
if project.commtrack_enabled:
supply_reports = (
commtrack.SimplifiedInventoryReport,
commtrack.InventoryReport,
commtrack.CurrentStockStatusReport,
commtrack.StockStatusMapReport,
)
if not should_use_sql_backend(project):
supply_reports = supply_reports + (
commtrack.ReportingRatesReport,
commtrack.ReportingStatusMapReport,
)
reports.insert(0, (ugettext_lazy("CommCare Supply"), supply_reports))
if project.has_careplan:
from corehq.apps.app_manager.models import CareplanConfig
config = CareplanConfig.for_domain(project.name)
if config:
cp_reports = tuple(make_careplan_reports(config))
reports.insert(0, (ugettext_lazy("Care Plans"), cp_reports))
reports = list(_get_report_builder_reports(project)) + reports
from corehq.apps.accounting.utils import domain_has_privilege
messaging_reports = []
project_can_use_sms = domain_has_privilege(project.name, privileges.OUTBOUND_SMS)
if project_can_use_sms:
messaging_reports.extend([
sms.MessagesReport,
])
# always have these historical reports visible
messaging_reports.extend([
sms.MessagingEventsReport,
sms.MessageEventDetailReport,
sms.SurveyDetailReport,
sms.MessageLogReport,
sms.SMSOptOutReport,
ivr.CallReport,
ivr.ExpectedCallbackReport,
])
messaging_reports += getattr(Domain.get_module_by_name(project.name), 'MESSAGING_REPORTS', ())
messaging = (ugettext_lazy("Messaging"), messaging_reports)
reports.append(messaging)
reports.extend(_get_dynamic_reports(project))
return reports
