def _do_migration_and_assert_flags(self, domain): self.assertFalse(should_use_sql_backend(domain)) call_command('migrate_domain_from_couch_to_sql', domain, MIGRATE=True, no_input=True) self.assertTrue(should_use_sql_backend(domain))
def test_duplicate_form_published(self): form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id) orig_form = submit_form_locally(form_xml, domain=self.domain)[1] self.assertEqual(form_id, orig_form.form_id) self.assertEqual(1, len(self.form_accessors.get_all_form_ids_in_domain())) with process_kafka_changes(self.form_pillow): with process_couch_changes('DefaultChangeFeedPillow'): # post an exact duplicate dupe_form = submit_form_locally(form_xml, domain=self.domain)[1] self.assertTrue(dupe_form.is_duplicate) self.assertNotEqual(form_id, dupe_form.form_id) if should_use_sql_backend(self.domain): self.assertEqual(form_id, dupe_form.orig_id) # make sure changes made it to kafka dupe_form_meta = self.processor.changes_seen[0].metadata self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id) self.assertEqual(dupe_form.domain, dupe_form.domain) if should_use_sql_backend(self.domain): # sql domains also republish the original form to ensure that if the server crashed # in the processing of the form the first time that it is still sent to kafka orig_form_meta = self.processor.changes_seen[1].metadata self.assertEqual(orig_form.form_id, orig_form_meta.document_id) self.assertEqual(self.domain, orig_form_meta.domain) self.assertEqual(dupe_form.domain, dupe_form.domain)
def test_duplicate_form_published(self): form_id = uuid.uuid4().hex form_xml = get_simple_form_xml(form_id) orig_form = submit_form_locally(form_xml, domain=self.domain).xform self.assertEqual(form_id, orig_form.form_id) self.assertEqual(1, len(self.form_accessors.get_all_form_ids_in_domain())) with process_pillow_changes(self.form_pillow): with process_pillow_changes('DefaultChangeFeedPillow'): # post an exact duplicate dupe_form = submit_form_locally(form_xml, domain=self.domain).xform self.assertTrue(dupe_form.is_duplicate) self.assertNotEqual(form_id, dupe_form.form_id) if should_use_sql_backend(self.domain): self.assertEqual(form_id, dupe_form.orig_id) # make sure changes made it to kafka dupe_form_meta = self.processor.changes_seen[0].metadata self.assertEqual(dupe_form.form_id, dupe_form_meta.document_id) self.assertEqual(dupe_form.domain, dupe_form.domain) if should_use_sql_backend(self.domain): # sql domains also republish the original form to ensure that if the server crashed # in the processing of the form the first time that it is still sent to kafka orig_form_meta = self.processor.changes_seen[1].metadata self.assertEqual(orig_form.form_id, orig_form_meta.document_id) self.assertEqual(self.domain, orig_form_meta.domain) self.assertEqual(dupe_form.domain, dupe_form.domain)
def commit_migration(domain_name): domain = Domain.get_by_name(domain_name, strict=True) domain.use_sql_backend = True domain.save() clear_local_domain_sql_backend_override(domain_name) if not should_use_sql_backend(domain_name): Domain.get_by_name.clear(Domain, domain_name) assert should_use_sql_backend(domain_name)
def commit_migration(domain_name): domain = Domain.get_by_name(domain_name, strict=True) domain.use_sql_backend = True domain.save() clear_local_domain_sql_backend_override(domain_name) if not should_use_sql_backend(domain_name): Domain.get_by_name.clear(Domain, domain_name) assert should_use_sql_backend(domain_name) datadog_counter("commcare.couch_sql_migration.total_committed") _logger.info("committed migration for {}".format(domain))
def commit_migration(domain_name): domain_obj = Domain.get_by_name(domain_name, strict=True) domain_obj.use_sql_backend = True domain_obj.save() clear_local_domain_sql_backend_override(domain_name) if not should_use_sql_backend(domain_name): Domain.get_by_name.clear(Domain, domain_name) assert should_use_sql_backend(domain_name) datadog_counter("commcare.couch_sql_migration.total_committed") _logger.info("committed migration for {}".format(domain_name))
def commit_migration(domain_name): domain_obj = Domain.get_by_name(domain_name, strict=True) domain_obj.use_sql_backend = True domain_obj.save() clear_local_domain_sql_backend_override(domain_name) if not should_use_sql_backend(domain_name): Domain.get_by_name.clear(Domain, domain_name) assert should_use_sql_backend(domain_name), \ "could not set use_sql_backend for domain %s (try again)" % domain_name datadog_counter("commcare.couch_sql_migration.total_committed") log.info("committed migration for {}".format(domain_name))
def setUp(self): super(TestHelperFunctions, self).setUp() FormProcessorTestUtils.delete_all_cases_forms_ledgers() self.domain_name = uuid.uuid4().hex self.domain = create_domain(self.domain_name) self.assertFalse(should_use_sql_backend(self.domain_name))
def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return set_couch_sql_migration_started(domain) with SignalHandlerContext([signal.SIGTERM, signal.SIGINT], _get_sigterm_handler(domain)): do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write( "Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table([ 'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs' ], [(doc_type, ) + stat for doc_type, stat in stats.items()]) else: assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green( "Domain migrated: {}".format(domain)))
def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): self.stderr.write("{} migration is already in progress".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write("Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table(['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'], [ (doc_type,) + stat for doc_type, stat in stats.items() ]) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green("Domain migrated: {}".format(domain))) return True, None
def handle(self, domain, **options): if not options['noinput']: confirm = input(""" Are you sure you want to hard delete all forms and cases in domain "{}"? This operation is PERMANENT. Type the domain's name again to continue, or anything else to cancel: """.format(domain)) if confirm != domain: print("\n\t\tDomain deletion cancelled.") return assert should_use_sql_backend(domain) logger.info('Hard deleting forms...') deleted_sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain( domain) for form_id_chunk in chunked( with_progress_bar(deleted_sql_form_ids, stream=silence_during_tests()), 500): FormAccessorSQL.hard_delete_forms(domain, list(form_id_chunk), delete_attachments=True) logger.info('Hard deleting cases...') deleted_sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain( domain) for case_id_chunk in chunked( with_progress_bar(deleted_sql_case_ids, stream=silence_during_tests()), 500): CaseAccessorSQL.hard_delete_cases(domain, list(case_id_chunk)) logger.info('Done.')
def test_dry_run(self): self.assertFalse(should_use_sql_backend(self.domain_name)) call_command( 'migrate_domain_from_couch_to_sql', self.domain_name, MIGRATE=True, no_input=True, dry_run=True ) clear_local_domain_sql_backend_override(self.domain_name) with self.assertRaises(CommandError): call_command('migrate_domain_from_couch_to_sql', self.domain_name, COMMIT=True, no_input=True) self.assertFalse(Domain.get_by_name(self.domain_name).use_sql_backend) xml = """<?xml version="1.0" ?> <n0:registration xmlns:n0="http://openrosa.org/user/registration"> <username>W4</username> <password>2</password> <uuid>P8DU7OLHVLZXU21JR10H3W8J2</uuid> <date>2013-11-19</date> <registering_phone_id>8H1N48EFPF6PA4UOO8YGZ2KFZ</registering_phone_id> <user_data> <data key="user_type">standard</data> </user_data> </n0:registration> """ submit_form_locally(xml, self.domain_name) couch_form_ids = self._get_form_ids() self.assertEqual(1, len(couch_form_ids)) call_command('migrate_domain_from_couch_to_sql', self.domain_name, blow_away=True, no_input=True) self.assertFalse(Domain.get_by_name(self.domain_name).use_sql_backend)
def handle(self, domain, action, **options): if should_use_sql_backend(domain): raise CommandError( 'It looks like {} has already been migrated.'.format(domain)) for opt in [ "no_input", "verbose", "state_dir", "live_migrate", "diff_process", "rebuild_state", ]: setattr(self, opt, options[opt]) if self.no_input and not settings.UNIT_TESTING: raise CommandError('--no-input only allowed for unit testing') if action != MIGRATE and self.live_migrate: raise CommandError("--live only allowed with `MIGRATE`") if action != MIGRATE and self.rebuild_state: raise CommandError("--rebuild-state only allowed with `MIGRATE`") if action != STATS and self.verbose: raise CommandError("--verbose only allowed for `stats`") assert Domain.get_by_name(domain), f'Unknown domain "{domain}"' slug = f"{action.lower()}-{domain}" setup_logging(self.state_dir, slug, options['debug']) getattr(self, "do_" + action)(domain)
def migrate_domain(self, domain, state_dir): if should_use_sql_backend(domain): log.info("{} already on the SQL backend\n".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): log.error("{} migration is in progress\n".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain, self.live_migrate) try: do_couch_to_sql_migration( domain, state_dir, with_progress=self.live_migrate, live_migrate=self.live_migrate, ) except MigrationRestricted as err: log.error("migration restricted: %s", err) set_couch_sql_migration_not_started(domain) return False, str(err) has_diffs = self.check_diffs(domain, state_dir) if self.live_migrate: return True, None if has_diffs: self.abort(domain, state_dir) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) log.info("Domain migrated: {}\n".format(domain)) return True, None
def handle(self, user, **options): try: self.user = CouchUser.get_by_username(user) if not self.user: self.user = CouchUser.get(user) except ResourceNotFound: print("Could not find user {}".format(user)) return if not isinstance(self.user, CommCareUser): print ("Sorry, the user you specify has to be a mobile worker. " "This changed when delete_cases was refactored to use " "cases_by_owner/view instead of case/by_owner. " "The new view needs an explicit domain, " "and I didn't implement that for WebUsers who can belong " "to multiple domains, but usually do not own cases.") exit(1) self.domain = self.user.domain if should_use_sql_backend(self.domain): raise CommandError('This command only works for couch-based domains.') if not options.get('no_prompt'): msg = "Delete all cases owned by {}? (y/n)\n".format( self.user.username, ) if not input(msg) == 'y': print("cancelling") return self.delete_all() print("Cases successfully deleted, you monster!")
def migrate_domain(self, domain, state_dir): if should_use_sql_backend(domain): log.info("{} already on the SQL backend\n".format(domain)) return if couch_sql_migration_in_progress(domain, include_dry_runs=True): log.error("{} migration is in progress\n".format(domain)) raise Incomplete("in progress") set_couch_sql_migration_started(domain, self.live_migrate) try: do_couch_to_sql_migration( domain, state_dir, with_progress=self.live_migrate, live_migrate=self.live_migrate, ) except MigrationRestricted as err: log.error("migration restricted: %s", err) set_couch_sql_migration_not_started(domain) raise Incomplete(str(err)) if self.live_migrate: self.finish_live_migration_if_possible(domain, state_dir) elif self.has_diffs(domain, state_dir): self.abort(domain, state_dir) raise Incomplete("has diffs") assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) log.info(f"Domain migrated: {domain}\n")
def migrate_domain(self, domain): if should_use_sql_backend(domain): log.error("{} already on the SQL backend\n".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): log.error("{} migration is already in progress\n".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: lines = ["Migration has diffs: {}".format(domain)] class stream: write = lines.append writer = SimpleTableWriter(stream, TableRowFormatter([30, 10, 10, 10, 10])) writer.write_table( ['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'], [(doc_type,) + stat for doc_type, stat in stats.items()], ) log.error("\n".join(lines)) self.abort(domain) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) log.info("Domain migrated: {}\n".format(domain)) return True, None
def handle(self, action, domain, **options): if should_use_sql_backend(domain): raise CommandError(f'It looks like {domain} has already been migrated.') for opt in [ "no_input", "debug", "state_path", "select", "stop", "changes", "batch_size", "reset", ]: setattr(self, opt, options[opt]) if self.no_input and not settings.UNIT_TESTING: raise CommandError('--no-input only allowed for unit testing') if self.changes and action != SHOW: raise CommandError('--changes only allowed with "show" action') if self.reset: if action == SHOW: raise CommandError(f'invalid action for --reset: {action}') self.do_reset(action, domain) return if action != SHOW: assert Domain.get_by_name(domain), f'Unknown domain "{domain}"' do_action = getattr(self, "do_" + action) msg = do_action(domain) if msg: sys.exit(msg)
def handle(self, user, **options): try: self.user = CouchUser.get_by_username(user) if not self.user: self.user = CouchUser.get(user) except ResourceNotFound: print("Could not find user {}".format(user)) return if not isinstance(self.user, CommCareUser): print("Sorry, the user you specify has to be a mobile worker. " "This changed when delete_cases was refactored to use " "cases_by_owner/view instead of case/by_owner. " "The new view needs an explicit domain, " "and I didn't implement that for WebUsers who can belong " "to multiple domains, but usually do not own cases.") exit(1) self.domain = self.user.domain if should_use_sql_backend(self.domain): raise CommandError( 'This command only works for couch-based domains.') if not options.get('no_prompt'): msg = "Delete all cases owned by {}? (y/n)\n".format( self.user.username, ) if not raw_input(msg) == 'y': print("cancelling") return self.delete_all() print("Cases successfully deleted, you monster!")
def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): self.stderr.write( "{} migration is already in progress".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write( "Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table([ 'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs' ], [(doc_type, ) + stat for doc_type, stat in stats.items()]) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green("Domain migrated: {}".format(domain))) return True, None
def migrate_domain(self, domain, state_dir): if should_use_sql_backend(domain): log.info("{} already on the SQL backend\n".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): log.error("{} migration is in progress\n".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain) try: do_couch_to_sql_migration(domain, state_dir, with_progress=False) except MigrationRestricted as err: log.error("migration restricted: %s", err) set_couch_sql_migration_not_started(domain) return False, str(err) stats = get_diff_stats(domain, state_dir, self.strict) if stats: header = "Migration has diffs: {}".format(domain) log.error(format_diff_stats(stats, header)) self.abort(domain, state_dir) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) log.info("Domain migrated: {}\n".format(domain)) return True, None
def handle(self, *args, **options): if len(args) == 1: domain = args[0] since = None elif len(args) == 2: domain = args[0] since = string_to_datetime(args[1]) else: raise CommandError('Usage: %s\n%s' % (self.args, self.help)) if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') succeeded = [] failed = [] error_messages = defaultdict(lambda: 0) for form in iter_problem_forms(domain, since): print "%s\t%s\t%s\t%s\t%s" % (form._id, form.received_on, form.xmlns, form.get_data('form/meta/username'), form.problem.strip()) if not options["dryrun"]: try: reprocess_form_cases(form) except Exception, e: failed.append(form._id) error_messages[str(e)] += 1 else: succeeded.append(form._id)
def property_changed_in_action(domain, case_transaction, case_id, case_property_name): from casexml.apps.case.xform import get_case_updates PropertyChangedInfo = namedtuple("PropertyChangedInfo", 'transaction new_value modified_on') include_create_fields = case_property_name in ['owner_id', 'name', 'external_id'] if not should_use_sql_backend(domain): # couch domains return 2 transactions for case properties created in a create form if case_transaction.is_case_create and not include_create_fields: return False case_updates = get_case_updates(case_transaction.form) actions = [] for update in case_updates: if update.id == case_id: actions.append((update.modified_on_str, update.get_update_action(), case_transaction)) if include_create_fields and case_transaction.is_case_create: actions.append((update.modified_on_str, update.get_create_action(), case_transaction)) for (modified_on, action, case_transaction) in actions: if action: property_changed = action.dynamic_properties.get(case_property_name) if include_create_fields and not property_changed: property_changed = getattr(action, case_property_name, None) if property_changed is not None: return PropertyChangedInfo(case_transaction, property_changed, modified_on) return False
def _get_form_ids(self, domain): if should_use_sql_backend(domain): problem_ids = FormAccessorSQL.get_form_ids_in_domain_by_type( domain, 'XFormError') else: problem_ids = get_form_ids_by_type(domain, 'XFormError') return problem_ids
def handle(self, *args, **options): if len(args) == 1: domain = args[0] since = None elif len(args) == 2: domain = args[0] since = string_to_datetime(args[1]) else: raise CommandError('Usage: %s\n%s' % (self.args, self.help)) if should_use_sql_backend(domain): raise CommandError( 'This command only works for couch-based domains.') succeeded = [] failed = [] error_messages = defaultdict(lambda: 0) for form in iter_problem_forms(domain, since): print "%s\t%s\t%s\t%s\t%s" % ( form._id, form.received_on, form.xmlns, form.get_data('form/meta/username'), form.problem.strip()) if not options["dryrun"]: try: reprocess_form_cases(form) except Exception, e: failed.append(form._id) error_messages[str(e)] += 1 else: succeeded.append(form._id)
def resave_form(domain, form): from corehq.form_processor.utils import should_use_sql_backend from corehq.form_processor.change_publishers import publish_form_saved from couchforms.models import XFormInstance if should_use_sql_backend(domain): publish_form_saved(form) else: XFormInstance.get_db().save_doc(form.to_json())
def print_migration(self, item): started = item.started_on migrated = should_use_sql_backend(item.domain) print(" {}{}{}".format( item.domain, started.strftime(" (%Y-%m-%d)") if started else "", " - already migrated? (bad state)" if migrated else "", ))
def _assert_no_migration_restrictions(self, domain_name): assert should_use_sql_backend(domain_name) assert not COUCH_SQL_MIGRATION_BLACKLIST.enabled( domain_name, NAMESPACE_DOMAIN) assert not any( custom_report_domain == domain_name for custom_report_domain in settings.DOMAIN_MODULE_MAP.keys()) assert not REMINDERS_MIGRATION_IN_PROGRESS.enabled(domain_name)
def resave_case(domain, case, send_post_save_signal=True): from corehq.form_processor.change_publishers import publish_case_saved if should_use_sql_backend(domain): publish_case_saved(case, send_post_save_signal) else: if send_post_save_signal: case.save() else: CommCareCase.get_db().save_doc(case._doc) # don't just call save to avoid signals
def get_case_ids_for_messaging_rule(domain, case_type): if not should_use_sql_backend(domain): return CaseAccessors(domain).get_case_ids_in_domain(case_type) else: return run_query_across_partitioned_databases( CommCareCaseSQL, Q(domain=domain, type=case_type, deleted=False), values=['case_id'] )
def setUp(self): super(BaseMigrationTestCase, self).setUp() FormProcessorTestUtils.delete_all_cases_forms_ledgers() self.domain_name = uuid.uuid4().hex self.domain = create_domain(self.domain_name) # all new domains are set complete when they are created DomainMigrationProgress.objects.filter(domain=self.domain_name).delete() self.assertFalse(should_use_sql_backend(self.domain_name))
def handle(self, *args, **options): if len(args) < 2: raise CommandError('Usage is copy_case, %s' % self.args) source_couch = CouchConfig(args[0]) case_id = args[1] doc_ids = [case_id] domain = args[2] if len(args) > 2 else None if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') def _migrate_case(case_id): print 'getting case %s' % case_id case = CommCareCase.wrap(source_couch.get_db_for_class(CommCareCase).get(case_id)) original_domain = case.domain if domain is not None: case.domain = domain case.save(force_update=True) return case, original_domain case, orig_domain = _migrate_case(case_id) print 'copying %s parent cases' % len(case.indices) for index in case.indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) # hack, set the domain back to make sure we get the reverse indices correctly case.domain = orig_domain with OverrideDB(CommCareCase, source_couch.get_db_for_class(CommCareCase)): child_indices = get_reverse_indices(case) print 'copying %s child cases' % len(child_indices) for index in child_indices: _migrate_case(index.referenced_id) doc_ids.append(index.referenced_id) print 'copying %s xforms' % len(case.xform_ids) def form_wrapper(row): doc = row['doc'] doc.pop('_attachments', None) doc.pop('external_blobs', None) return XFormInstance.wrap(doc) xforms = source_couch.get_db_for_class(XFormInstance).all_docs( keys=case.xform_ids, include_docs=True, wrapper=form_wrapper, ).all() for form in xforms: if domain is not None: form.domain = domain form.save(force_update=True) print 'saved %s' % form._id doc_ids.append(form._id) if options['postgres_db']: copy_postgres_data_for_docs(options['postgres_db'], doc_ids)
def handle(self, domain, **options): if should_use_sql_backend(domain): raise CommandError('It looks like {} has already been migrated.'.format(domain)) self.no_input = options.pop('no_input', False) self.debug = options.pop('debug', False) self.dry_run = options.pop('dry_run', False) if self.no_input and not settings.UNIT_TESTING: raise CommandError('no-input only allowed for unit testing') setup_logging(options['log_dir']) if options['MIGRATE']: self.require_only_option('MIGRATE', options) if options.get('run_timestamp'): if not couch_sql_migration_in_progress(domain): raise CommandError("Migration must be in progress if run_timestamp is passed in") else: set_couch_sql_migration_started(domain, self.dry_run) do_couch_to_sql_migration( domain, with_progress=not self.no_input, debug=self.debug, run_timestamp=options.get('run_timestamp')) has_diffs = self.print_stats(domain, short=True, diffs_only=True) if has_diffs: print("\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n") if options['blow_away']: self.require_only_option('blow_away', options) if not self.no_input: _confirm( "This will delete all SQL forms and cases for the domain {}. " "Are you sure you want to continue?".format(domain) ) set_couch_sql_migration_not_started(domain) blow_away_migration(domain) if options['stats_short'] or options['stats_long']: self.print_stats(domain, short=options['stats_short']) if options['show_diffs']: self.show_diffs(domain) if options['COMMIT']: self.require_only_option('COMMIT', options) if not couch_sql_migration_in_progress(domain, include_dry_runs=False): raise CommandError("cannot commit a migration that is not in state in_progress") if not self.no_input: _confirm( "This will convert the domain to use the SQL backend and" "allow new form submissions to be processed. " "Are you sure you want to do this for domain '{}'?".format(domain) ) set_couch_sql_migration_complete(domain)
def handle(self, domain, action, **options): if action != STATS and should_use_sql_backend(domain): raise CommandError( 'It looks like {} has already been migrated.'.format(domain)) if options["patch"]: if options["forms"]: raise CommandError( "--patch and --forms=... are mutually exclusive") if options["case_diff"] != "after": raise CommandError( "--patch and --case-diff=... are mutually exclusive") options["forms"] = "missing" options["case_diff"] = "patch" for opt in [ "no_input", "verbose", "state_dir", "live_migrate", "finish", "case_diff", "rebuild_state", "stop_on_error", "forms", "rewind", "missing_docs", ]: setattr(self, opt, options[opt]) if self.no_input and not settings.UNIT_TESTING: raise CommandError('--no-input only allowed for unit testing') if action != MIGRATE and self.live_migrate: raise CommandError(f"{action} --live not allowed") if action != MIGRATE and self.finish: raise CommandError(f"{action} --finish not allowed") if action != MIGRATE and self.rebuild_state: raise CommandError("--rebuild-state only allowed with `MIGRATE`") if action != MIGRATE and self.forms: raise CommandError("--forms only allowed with `MIGRATE`") if action != MIGRATE and self.stop_on_error: raise CommandError("--stop-on-error only allowed with `MIGRATE`") if action != STATS and self.verbose: raise CommandError("--verbose only allowed for `stats`") if action not in [MIGRATE, STATS] and self.missing_docs != CACHED: raise CommandError(f"{action} --missing-docs not allowed") if action != REWIND and self.rewind: raise CommandError("--to=... only allowed for `rewind`") if self.case_diff == "patch" and self.forms not in [None, "missing"]: raise CommandError( f"not supported: --case-diff=patch --forms={self.forms}") assert Domain.get_by_name(domain), f'Unknown domain "{domain}"' slug = f"{action.lower()}-{domain}" setup_logging(self.state_dir, slug, options['debug']) getattr(self, "do_" + action)(domain)
def _check_for_migration_restrictions(self, domain_name): msgs = [] if not should_use_sql_backend(domain_name): msgs.append("does not have SQL backend enabled") if COUCH_SQL_MIGRATION_BLACKLIST.enabled(domain_name, NAMESPACE_DOMAIN): msgs.append("is blacklisted") if domain_name in settings.DOMAIN_MODULE_MAP: msgs.append("has custom reports") if msgs: raise MigrationRestricted("{}: {}".format(domain_name, "; ".join(msgs)))
def get_document_store(domain, doc_type): use_sql = should_use_sql_backend(domain) if use_sql and doc_type == 'XFormInstance': return ReadonlyFormDocumentStore(domain) elif use_sql and doc_type == 'CommCareCase': return ReadonlyCaseDocumentStore(domain) else: # all other types still live in couchdb return CouchDocumentStore(couch_db=get_db_by_doc_type(doc_type), domain=domain, doc_type=doc_type)
def __init__(self, domain, with_progress=True, debug=False): from corehq.apps.tzmigration.planning import DiffDB assert should_use_sql_backend(domain) self.with_progress = with_progress self.debug = debug self.domain = domain db_filepath = get_diff_db_filepath(domain) self.diff_db = DiffDB.init(db_filepath) self.errors_with_normal_doc_type = []
def __init__(self, domain, with_progress=True, debug=False): from corehq.apps.tzmigration.planning import DiffDB assert should_use_sql_backend(domain) self.with_progress = with_progress self.debug = debug self.domain = domain db_filepath = get_diff_db_filepath(domain) self.diff_db = DiffDB.init(db_filepath) self.errors_with_normal_doc_type = [] self.forms_that_touch_cases_without_actions = set()
def handle(self, domain, reason, **options): if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') ids = get_case_ids_in_domain(domain) for count, case_id in enumerate(ids): try: rebuild_case_from_forms(domain, case_id, RebuildWithReason(reason=reason)) if count % 100 == 0: print('rebuilt %s/%s cases' % (count, len(ids))) except Exception as e: logging.exception("couldn't rebuild case {id}. {msg}".format(id=case_id, msg=str(e)))
def get_document_store(domain, doc_type): use_sql = should_use_sql_backend(domain) if use_sql and doc_type == 'XFormInstance': return ReadonlyFormDocumentStore(domain) elif use_sql and doc_type == 'CommCareCase': return ReadonlyCaseDocumentStore(domain) else: # all other types still live in couchdb return CouchDocumentStore( couch_db=get_db_by_doc_type(doc_type), domain=domain, doc_type=doc_type )
def setUp(self): super(BaseMigrationTestCase, self).setUp() with trap_extra_setup(AttributeError, msg="S3_BLOB_DB_SETTINGS not configured"): config = settings.S3_BLOB_DB_SETTINGS self.s3db = TemporaryS3BlobDB(config) assert get_blob_db() is self.s3db, (get_blob_db(), self.s3db) FormProcessorTestUtils.delete_all_cases_forms_ledgers() self.domain_name = uuid.uuid4().hex self.domain = create_domain(self.domain_name) # all new domains are set complete when they are created DomainMigrationProgress.objects.filter(domain=self.domain_name).delete() self.assertFalse(should_use_sql_backend(self.domain_name))
def handle(self, domain, doc_type, **options): csv = options.get('csv') startdate = options.get('start') enddate = options.get('end') form_doc_types = doc_types() if startdate or enddate: if doc_type in CASE_DOC_TYPES or doc_type in form_doc_types: if not should_use_sql_backend(domain): raise CommandError("Date filtering not supported for Couch domains") if startdate and enddate and enddate <= startdate: raise CommandError("enddate must be after startdate") handlers = { 'CommCareCase': compare_cases, 'CommCareCase-Deleted': compare_cases, 'CommCareUser': _compare_users, 'CommCareUser-Deleted': _compare_users, 'WebUser': _compare_users, } handlers.update({doc_type: compare_xforms for doc_type in form_doc_types}) try: primary_count, es_count, primary_only, es_only = handlers[doc_type](domain, doc_type, startdate, enddate) except KeyError: raise CommandError('Unsupported doc type. Use on of: {}'.format(', '.join(handlers))) if csv: row_formatter = CSVRowFormatter() else: row_formatter = TableRowFormatter([50, 50]) date_range_output = '' if startdate or enddate: end = (enddate or datetime.utcnow().date()).strftime(DATE_FORMAT) start = startdate.strftime(DATE_FORMAT) date_range_output = ' (Between {} and {})'.format(start, end) print("\nDoc ID analysis for {}{}\n".format(doc_type, date_range_output)) print("Primary Count: {}".format(primary_count)) print("ES Count: {}\n".format(es_count)) writer = SimpleTableWriter(self.stdout, row_formatter) writer.write_table( ['Only in Primary', 'Only in ES'], zip_longest(primary_only, es_only, fillvalue='') )
def _blow_away_migration(domain): assert not should_use_sql_backend(domain) delete_diff_db(domain) for doc_type in doc_types(): sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type) FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False) sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain) FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False) sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain) CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids) sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain) CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
def sync_cases(self, domain): db_aliases = get_db_aliases_for_partitioned_query() db_aliases.sort() if should_use_sql_backend(domain): case_accessor = CaseReindexAccessor(domain) case_ids = (case.case_id for case in iter_all_rows(case_accessor)) else: changes = _get_case_iterator(domain).iter_all_changes() case_ids = (case.id for case in changes) next_event = time.time() + 10 for i, case_id in enumerate(case_ids): sync_case_for_messaging.delay(domain, case_id) if time.time() > next_event: print("Queued %d cases for domain %s" % (i + 1, domain)) next_event = time.time() + 10
def handle(self, domain, csv_file, **options): self.domain = domain if not should_use_sql_backend(domain): print("This domain doesn't use SQL backend, exiting!") return current_date = self.first_form_received_on() if not current_date: print("No submissions in this domain yet, exiting!") return with open(csv_file, "w", encoding='utf-8') as csv_file: field_names = ('date', 'doc_type', 'in_sql', 'in_es', 'diff') csv_writer = csv.DictWriter(csv_file, field_names, extrasaction='ignore') csv_writer.writeheader() while current_date <= datetime.today(): cases_in_sql = self._get_sql_cases_modified_on_date(current_date) cases_in_es = self._get_es_cases_modified_on_date(current_date) properties = { "date": current_date, "doc_type": "CommCareCase", "in_sql": cases_in_sql, "in_es": cases_in_es, "diff": cases_in_sql - cases_in_es, } csv_writer.writerow(properties) print(properties) forms_in_sql = self._get_sql_forms_received_on_date(current_date) forms_in_es = self._get_es_forms_received_on_date(current_date) properties = { "date": current_date, "doc_type": "XFormInstance", "in_sql": forms_in_sql, "in_es": forms_in_es, "diff": forms_in_sql - forms_in_es } csv_writer.writerow(properties) print(properties) current_date += relativedelta(months=1)
def handle(self, domain, **options): if should_use_sql_backend(domain): raise CommandError('This command only works for couch-based domains.') filepath = get_planning_db_filepath(domain) self.stdout.write('Using file {}\n'.format(filepath)) if options['BEGIN']: self.require_only_option('BEGIN', options) set_tz_migration_started(domain) if options['ABORT']: self.require_only_option('ABORT', options) set_tz_migration_not_started(domain) if options['blow_away']: delete_planning_db(domain) self.stdout.write('Removed file {}\n'.format(filepath)) if options['prepare']: self.planning_db = prepare_planning_db(domain) self.stdout.write('Created and loaded file {}\n'.format(filepath)) else: self.planning_db = get_planning_db(domain) if options['COMMIT']: self.require_only_option('COMMIT', options) assert get_tz_migration_status(domain, strict=True) == MigrationStatus.IN_PROGRESS commit_plan(domain, self.planning_db) set_tz_migration_complete(domain) if options['prepare_case_json']: prepare_case_json(self.planning_db) if options['stats']: self.valiate_forms_and_cases(domain) if options['show_diffs']: self.show_diffs() if options['play']: from corehq.apps.tzmigration.planning import * session = self.planning_db.Session() # noqa try: import ipdb as pdb except ImportError: import pdb pdb.set_trace()
def handle_label(self, domain, **options): if should_use_sql_backend(domain): raise CommandError(u'It looks like {} has already been migrated.'.format(domain)) self.no_input = options.pop('no_input', False) self.debug = options.pop('debug', False) if self.no_input and not settings.UNIT_TESTING: raise CommandError('no-input only allowed for unit testing') if options['MIGRATE']: self.require_only_option('MIGRATE', options) set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=not self.no_input, debug=self.debug) has_diffs = self.print_stats(domain, short=True, diffs_only=True) if has_diffs: print "\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n" if options['blow_away']: self.require_only_option('blow_away', options) if not self.no_input: _confirm( "This will delete all SQL forms and cases for the domain {}. " "Are you sure you want to continue?".format(domain) ) set_couch_sql_migration_not_started(domain) _blow_away_migration(domain) if options['stats_short'] or options['stats_long']: self.print_stats(domain, short=options['stats_short']) if options['show_diffs']: self.show_diffs(domain) if options['COMMIT']: self.require_only_option('COMMIT', options) assert couch_sql_migration_in_progress(domain) if not self.no_input: _confirm( "This will allow convert the domain to use the SQL backend and" "allow new form submissions to be processed. " "Are you sure you want to do this for domain '{}'?".format(domain) ) set_couch_sql_migration_complete(domain)
def filter_cases(request, domain, app_id, module_id, parent_id=None): app = Application.get(app_id) module = app.get_module(module_id) auth_cookie = request.COOKIES.get('sessionid') requires_parent_cases = string_to_boolean(request.GET.get('requires_parent_cases', 'false')) xpath = EntriesHelper.get_filter_xpath(module) instances = get_instances_for_module(app, module, additional_xpaths=[xpath]) extra_instances = [{'id': inst.id, 'src': inst.src} for inst in instances] accessor = CaseAccessors(domain) # touchforms doesn't like this to be escaped xpath = HTMLParser.HTMLParser().unescape(xpath) case_type = module.case_type if xpath or should_use_sql_backend(domain): # if we need to do a custom filter, send it to touchforms for processing additional_filters = { "properties/case_type": case_type, "footprint": True } helper = BaseSessionDataHelper(domain, request.couch_user) result = helper.filter_cases(xpath, additional_filters, DjangoAuth(auth_cookie), extra_instances=extra_instances) if result.get('status', None) == 'error': code = result.get('code', 500) message = result.get('message', _("Something went wrong filtering your cases.")) if code == 500: notify_exception(None, message=message) return json_response(message, status_code=code) case_ids = result.get("cases", []) else: # otherwise just use our built in api with the defaults case_ids = [res.id for res in get_filtered_cases( domain, status=CASE_STATUS_OPEN, case_type=case_type, user_id=request.couch_user._id, footprint=True, ids_only=True, )] cases = accessor.get_cases(case_ids) if parent_id: cases = filter(lambda c: c.parent and c.parent.case_id == parent_id, cases) # refilter these because we might have accidentally included footprint cases # in the results from touchforms. this is a little hacky but the easiest # (quick) workaround. should be revisted when we optimize the case list. cases = filter(lambda c: c.type == case_type, cases) cases = [c.to_api_json(lite=True) for c in cases if c] response = {'cases': cases} if requires_parent_cases: # Subtract already fetched cases from parent list parent_ids = set(map(lambda c: c['indices']['parent']['case_id'], cases)) - \ set(map(lambda c: c['case_id'], cases)) parents = accessor.get_cases(list(parent_ids)) parents = [c.to_api_json(lite=True) for c in parents] response.update({'parents': parents}) return json_response(response)
def REPORTS(project): from corehq.apps.reports.standard.cases.basic import CaseListReport report_set = None if project.report_whitelist: report_set = set(project.report_whitelist) reports = [] reports.extend(_get_configurable_reports(project)) monitoring_reports = ( monitoring.WorkerActivityReport, monitoring.DailyFormStatsReport, monitoring.SubmissionsByFormReport, monitoring.FormCompletionTimeReport, monitoring.CaseActivityReport, monitoring.FormCompletionVsSubmissionTrendsReport, monitoring.WorkerActivityTimes, ProjectHealthDashboard, ) inspect_reports = [ inspect.SubmitHistory, CaseListReport, OdmExportReport, ] if toggles.CASE_LIST_EXPLORER.enabled(project.name): inspect_reports.append(CaseListExplorer) deployments_reports = ( deployments.ApplicationStatusReport, deployments.AggregateUserStatusReport, receiverwrapper.SubmissionErrorReport, phonelog.DeviceLogDetailsReport, deployments.ApplicationErrorReport, ) monitoring_reports = _filter_reports(report_set, monitoring_reports) inspect_reports = _filter_reports(report_set, inspect_reports) deployments_reports = _filter_reports(report_set, deployments_reports) reports.extend([ (ugettext_lazy("Monitor Workers"), monitoring_reports), (ugettext_lazy("Inspect Data"), inspect_reports), (ugettext_lazy("Manage Deployments"), deployments_reports), ]) if project.commtrack_enabled: supply_reports = ( commtrack.SimplifiedInventoryReport, commtrack.InventoryReport, commtrack.CurrentStockStatusReport, commtrack.StockStatusMapReport, ) if not should_use_sql_backend(project): supply_reports = supply_reports + ( commtrack.ReportingRatesReport, commtrack.ReportingStatusMapReport, ) supply_reports = _filter_reports(report_set, supply_reports) reports.insert(0, (ugettext_lazy("CommCare Supply"), supply_reports)) reports = list(_get_report_builder_reports(project)) + reports from corehq.apps.accounting.utils import domain_has_privilege messaging_reports = [] project_can_use_sms = domain_has_privilege(project.name, privileges.OUTBOUND_SMS) if project_can_use_sms: messaging_reports.extend([ sms.MessagesReport, ]) # always have these historical reports visible messaging_reports.extend([ sms.MessagingEventsReport, sms.MessageEventDetailReport, sms.SurveyDetailReport, sms.MessageLogReport, sms.SMSOptOutReport, ivr.CallReport, ivr.ExpectedCallbackReport, sms.PhoneNumberReport, sms.ScheduleInstanceReport, ]) messaging_reports += getattr(Domain.get_module_by_name(project.name), 'MESSAGING_REPORTS', ()) messaging_reports = _filter_reports(report_set, messaging_reports) messaging = (ugettext_lazy("Messaging"), messaging_reports) reports.append(messaging) reports.extend(_get_dynamic_reports(project)) return reports
def _assert_no_migration_restrictions(self, domain_name): assert should_use_sql_backend(domain_name) assert not COUCH_SQL_MIGRATION_BLACKLIST.enabled(domain_name, NAMESPACE_DOMAIN) assert not any(custom_report_domain == domain_name for custom_report_domain in settings.DOMAIN_MODULE_MAP.keys()) assert not REMINDERS_MIGRATION_IN_PROGRESS.enabled(domain_name)
def _process_form(request, domain, app_id, user_id, authenticated, auth_cls=AuthContext): metric_tags = [ 'backend:sql' if should_use_sql_backend(domain) else 'backend:couch', 'domain:{}'.format(domain), ] if should_ignore_submission(request): # silently ignore submission if it meets ignore-criteria response = openrosa_response.SUBMISSION_IGNORED_RESPONSE _record_metrics(metric_tags, 'ignored', response) return response if toggles.FORM_SUBMISSION_BLACKLIST.enabled(domain): response = openrosa_response.BLACKLISTED_RESPONSE _record_metrics(metric_tags, 'blacklisted', response) return response with TimingContext() as timer: try: instance, attachments = couchforms.get_instance_and_attachment(request) except MultimediaBug as e: try: instance = request.FILES[MAGIC_PROPERTY].read() xform = convert_xform_to_json(instance) meta = xform.get("meta", {}) except: meta = {} return _submission_error( request, "Received a submission with POST.keys()", MULTIMEDIA_SUBMISSION_ERROR_COUNT, metric_tags, domain, app_id, user_id, authenticated, meta, ) app_id, build_id = get_app_and_build_ids(domain, app_id) submission_post = SubmissionPost( instance=instance, attachments=attachments, domain=domain, app_id=app_id, build_id=build_id, auth_context=auth_cls( domain=domain, user_id=user_id, authenticated=authenticated, ), location=couchforms.get_location(request), received_on=couchforms.get_received_on(request), date_header=couchforms.get_date_header(request), path=couchforms.get_path(request), submit_ip=couchforms.get_submit_ip(request), last_sync_token=couchforms.get_last_sync_token(request), openrosa_headers=couchforms.get_openrosa_headers(request), ) try: result = submission_post.run() except XFormLockError as err: return _submission_error( request, "XFormLockError: %s" % err, XFORM_LOCKED_COUNT, metric_tags, domain, app_id, user_id, authenticated, status=423, notify=False, ) response = result.response if response.status_code == 400: logging.error( 'Status code 400 for a form submission. ' 'Response is: \n{0}\n' ) _record_metrics(metric_tags, result.submission_type, response, result, timer) return response
def REPORTS(project): from corehq.apps.reports.standard.cases.basic import CaseListReport from corehq.apps.reports.standard.cases.careplan import make_careplan_reports reports = [] reports.extend(_get_configurable_reports(project)) reports.extend([ (ugettext_lazy("Monitor Workers"), ( monitoring.WorkerActivityReport, monitoring.DailyFormStatsReport, monitoring.SubmissionsByFormReport, monitoring.FormCompletionTimeReport, monitoring.CaseActivityReport, monitoring.FormCompletionVsSubmissionTrendsReport, monitoring.WorkerActivityTimes, ProjectHealthDashboard, )), (ugettext_lazy("Inspect Data"), ( inspect.SubmitHistory, CaseListReport, OdmExportReport, )), (ugettext_lazy("Manage Deployments"), ( deployments.ApplicationStatusReport, receiverwrapper.SubmissionErrorReport, phonelog.DeviceLogDetailsReport, deployments.SyncHistoryReport, deployments.ApplicationErrorReport, )), ]) if project.commtrack_enabled: supply_reports = ( commtrack.SimplifiedInventoryReport, commtrack.InventoryReport, commtrack.CurrentStockStatusReport, commtrack.StockStatusMapReport, ) if not should_use_sql_backend(project): supply_reports = supply_reports + ( commtrack.ReportingRatesReport, commtrack.ReportingStatusMapReport, ) reports.insert(0, (ugettext_lazy("CommCare Supply"), supply_reports)) if project.has_careplan: from corehq.apps.app_manager.models import CareplanConfig config = CareplanConfig.for_domain(project.name) if config: cp_reports = tuple(make_careplan_reports(config)) reports.insert(0, (ugettext_lazy("Care Plans"), cp_reports)) reports = list(_get_report_builder_reports(project)) + reports from corehq.apps.accounting.utils import domain_has_privilege messaging_reports = [] project_can_use_sms = domain_has_privilege(project.name, privileges.OUTBOUND_SMS) if project_can_use_sms: messaging_reports.extend([ sms.MessagesReport, ]) # always have these historical reports visible messaging_reports.extend([ sms.MessagingEventsReport, sms.MessageEventDetailReport, sms.SurveyDetailReport, sms.MessageLogReport, sms.SMSOptOutReport, ivr.CallReport, ivr.ExpectedCallbackReport, ]) messaging_reports += getattr(Domain.get_module_by_name(project.name), 'MESSAGING_REPORTS', ()) messaging = (ugettext_lazy("Messaging"), messaging_reports) reports.append(messaging) reports.extend(_get_dynamic_reports(project)) return reports