def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): self.stderr.write("{} migration is already in progress".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write("Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table(['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'], [ (doc_type,) + stat for doc_type, stat in stats.items() ]) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green("Domain migrated: {}".format(domain))) return True, None
def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return True, None if couch_sql_migration_in_progress(domain, include_dry_runs=True): self.stderr.write( "{} migration is already in progress".format(domain)) return False, "in progress" set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write( "Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table([ 'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs' ], [(doc_type, ) + stat for doc_type, stat in stats.items()]) return False, "has diffs" assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green("Domain migrated: {}".format(domain))) return True, None
def handle(self, domain, **options): if should_use_sql_backend(domain): raise CommandError('It looks like {} has already been migrated.'.format(domain)) self.no_input = options.pop('no_input', False) self.debug = options.pop('debug', False) self.dry_run = options.pop('dry_run', False) if self.no_input and not settings.UNIT_TESTING: raise CommandError('no-input only allowed for unit testing') setup_logging(options['log_dir']) if options['MIGRATE']: self.require_only_option('MIGRATE', options) if options.get('run_timestamp'): if not couch_sql_migration_in_progress(domain): raise CommandError("Migration must be in progress if run_timestamp is passed in") else: set_couch_sql_migration_started(domain, self.dry_run) do_couch_to_sql_migration( domain, with_progress=not self.no_input, debug=self.debug, run_timestamp=options.get('run_timestamp')) has_diffs = self.print_stats(domain, short=True, diffs_only=True) if has_diffs: print("\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n") if options['blow_away']: self.require_only_option('blow_away', options) if not self.no_input: _confirm( "This will delete all SQL forms and cases for the domain {}. " "Are you sure you want to continue?".format(domain) ) set_couch_sql_migration_not_started(domain) blow_away_migration(domain) if options['stats_short'] or options['stats_long']: self.print_stats(domain, short=options['stats_short']) if options['show_diffs']: self.show_diffs(domain) if options['COMMIT']: self.require_only_option('COMMIT', options) if not couch_sql_migration_in_progress(domain, include_dry_runs=False): raise CommandError("cannot commit a migration that is not in state in_progress") if not self.no_input: _confirm( "This will convert the domain to use the SQL backend and" "allow new form submissions to be processed. " "Are you sure you want to do this for domain '{}'?".format(domain) ) set_couch_sql_migration_complete(domain)
def get_case_with_lock(self, case_id, lock=False, wrap=False): """ Get a case with option lock. If case not found in domains DB also check other DB and raise IllegalCaseId if found. :param case_id: ID of case to fetch :param lock: Get a Redis lock for the case. Returns None if False or case not found. :param wrap: Return wrapped case if True. (Couch only) :return: tuple(case, lock). Either could be None :raises: IllegalCaseId """ # check across Couch & SQL to ensure global uniqueness # check this domains DB first to support existing bad data from corehq.apps.couch_sql_migration.progress import couch_sql_migration_in_progress case, lock = self.processor.get_case_with_lock(case_id, lock, wrap) if case: return case, lock if not couch_sql_migration_in_progress( self.domain) and not settings.ENTERPRISE_MODE: # during migration we're copying from one DB to the other so this check will always fail if self.other_db_processor().case_exists(case_id): raise IllegalCaseId("Bad case id") return case, lock
def do_MIGRATE(self, domain): if self.finish: assert not self.live_migrate, "--live and --finish are mutually exclusive" elif not self.live_migrate: status = get_couch_sql_migration_status(domain) if status == MigrationStatus.DRY_RUN: log.info( "Continuing live migration. Use --finish to complete.") self.live_migrate = True if self.missing_docs == CACHED: self.missing_docs = RESUME if self.forms: if not couch_sql_migration_in_progress(domain): log.error("cannot migrate specific forms: migration is %s", get_couch_sql_migration_status(domain)) sys.exit(1) else: set_couch_sql_migration_started(domain, self.live_migrate) do_couch_to_sql_migration( domain, self.state_dir, with_progress=not self.no_input, live_migrate=self.live_migrate, case_diff=self.case_diff, rebuild_state=self.rebuild_state, stop_on_error=self.stop_on_error, forms=self.forms, ) has_diffs = self.print_stats(domain, short=True, diffs_only=True) if self.live_migrate: print("Live migration completed.") if has_diffs: print("\nRun `diff` or `stats [--verbose]` for more details.\n") sys.exit(1)
def get_case_with_lock(self, case_id, lock=False, wrap=False): """ Get a case with option lock. If case not found in domains DB also check other DB and raise IllegalCaseId if found. :param case_id: ID of case to fetch :param lock: Get a Redis lock for the case. Returns None if False or case not found. :param wrap: Return wrapped case if True. (Couch only) :return: tuple(case, lock). Either could be None :raises: IllegalCaseId """ # check across Couch & SQL to ensure global uniqueness # check this domains DB first to support existing bad data from corehq.apps.couch_sql_migration.progress import couch_sql_migration_in_progress case, lock = self.processor.get_case_with_lock(case_id, lock, wrap) if case: return case, lock if not couch_sql_migration_in_progress(self.domain) and not settings.ENTERPRISE_MODE: # during migration we're copying from one DB to the other so this check will always fail if self.other_db_processor().case_exists(case_id): raise IllegalCaseId("Bad case id") return case, lock
def migrate_domain(self, domain): if should_use_sql_backend(domain): self.stderr.write("{} already on the SQL backend".format(domain)) return set_couch_sql_migration_started(domain) with SignalHandlerContext([signal.SIGTERM, signal.SIGINT], _get_sigterm_handler(domain)): do_couch_to_sql_migration(domain, with_progress=False, debug=False) stats = self.get_diff_stats(domain) if stats: self.stderr.write( "Migration has diffs, aborting for domain {}".format(domain)) self.abort(domain) writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10])) writer.write_table([ 'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs' ], [(doc_type, ) + stat for doc_type, stat in stats.items()]) else: assert couch_sql_migration_in_progress(domain) set_couch_sql_migration_complete(domain) self.stdout.write(shell_green( "Domain migrated: {}".format(domain)))
def submit_system_action(name, args, args_json, domain): """Submit a system action to be recorded as a form Alias: `system_action.submit(...)` Record a system action (un/archive form, etc.) in the stream of submitted forms, which allows it to be reproduced during a migration, for example. :param name: system action name. :param args: list of action arguments, some of which may not be JSON-serializable. :param args_json: a JSON-serializable object having enough information to reconstruct `args` given the action name. :param domain: The domain in which to perform the action. """ from corehq.apps.hqcase.utils import submit_case_blocks if name not in _actions: raise ValueError("unknown system action: {}".format(name)) if couch_sql_migration_in_progress(domain): # record system action if migration in progress log.debug("submit %s %s in %s", name, args_json, domain) xml_name = escape(name) xml_args = escape(json.dumps(args_json)) submit_case_blocks( f"<name>{xml_name}</name><args>{xml_args}</args>", domain, xmlns=SYSTEM_ACTION_XMLNS, device_id=name, form_extras={"auth_context": SystemActionContext}, ) do_system_action(name, args)
def do_COMMIT(self, domain): if not couch_sql_migration_in_progress(domain, include_dry_runs=False): raise CommandError("cannot commit a migration that is not in state in_progress") if not self.no_input: _confirm( "This will convert the domain to use the SQL backend and" "allow new form submissions to be processed. " "Are you sure you want to do this for domain '{}'?".format(domain) ) set_couch_sql_migration_complete(domain)
def handle(self, domain, **options): check_only = options['check'] log("Handling new reminders migration for %s, --check option is %s" % (domain, check_only)) domain_obj = Domain.get_by_name(domain) if self.migration_already_done(domain_obj): return if not check_only: self.ensure_migration_flag_enabled(domain) if not check_only and couch_sql_migration_in_progress(domain): log("The Couch to SQL migration is in progress for this project, halting." ) self.ensure_migration_flag_disabled(domain) return handlers = self.get_handlers_to_migrate(domain) migrators, cannot_be_migrated = self.get_migrators(handlers) if cannot_be_migrated: return log("Migration can proceed") if check_only: return if not self.confirm( "Are you sure you want to start the migration? y/n "): log("Migrated halted") return self.migrate_handlers(migrators) self.refresh_instances(domain, migrators) log("\n") if not self.confirm("Ok to switch on new reminders? y/n "): log("Migrated halted") return self.switch_on_new_reminders(domain, migrators) self.ensure_migration_flag_disabled(domain) log("Migration completed.")
def handle_label(self, domain, **options): if should_use_sql_backend(domain): raise CommandError( u'It looks like {} has already been migrated.'.format(domain)) self.no_input = options.pop('no_input', False) self.debug = options.pop('debug', False) if self.no_input and not settings.UNIT_TESTING: raise CommandError('no-input only allowed for unit testing') if options['MIGRATE']: self.require_only_option('MIGRATE', options) set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=not self.no_input, debug=self.debug) has_diffs = self.print_stats(domain, short=True, diffs_only=True) if has_diffs: print "\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n" if options['blow_away']: self.require_only_option('blow_away', options) if not self.no_input: _confirm( "This will delete all SQL forms and cases for the domain {}. " "Are you sure you want to continue?".format(domain)) set_couch_sql_migration_not_started(domain) _blow_away_migration(domain) if options['stats_short'] or options['stats_long']: self.print_stats(domain, short=options['stats_short']) if options['show_diffs']: self.show_diffs(domain) if options['COMMIT']: self.require_only_option('COMMIT', options) assert couch_sql_migration_in_progress(domain) if not self.no_input: _confirm( "This will allow convert the domain to use the SQL backend and" "allow new form submissions to be processed. " "Are you sure you want to do this for domain '{}'?".format( domain)) set_couch_sql_migration_complete(domain)
def handle_label(self, domain, **options): if should_use_sql_backend(domain): raise CommandError(u'It looks like {} has already been migrated.'.format(domain)) self.no_input = options.pop('no_input', False) self.debug = options.pop('debug', False) if self.no_input and not settings.UNIT_TESTING: raise CommandError('no-input only allowed for unit testing') if options['MIGRATE']: self.require_only_option('MIGRATE', options) set_couch_sql_migration_started(domain) do_couch_to_sql_migration(domain, with_progress=not self.no_input, debug=self.debug) has_diffs = self.print_stats(domain, short=True, diffs_only=True) if has_diffs: print "\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n" if options['blow_away']: self.require_only_option('blow_away', options) if not self.no_input: _confirm( "This will delete all SQL forms and cases for the domain {}. " "Are you sure you want to continue?".format(domain) ) set_couch_sql_migration_not_started(domain) _blow_away_migration(domain) if options['stats_short'] or options['stats_long']: self.print_stats(domain, short=options['stats_short']) if options['show_diffs']: self.show_diffs(domain) if options['COMMIT']: self.require_only_option('COMMIT', options) assert couch_sql_migration_in_progress(domain) if not self.no_input: _confirm( "This will allow convert the domain to use the SQL backend and" "allow new form submissions to be processed. " "Are you sure you want to do this for domain '{}'?".format(domain) ) set_couch_sql_migration_complete(domain)
def _apply_case_update(self, case_update, xformdoc): sql_migration_in_progress = couch_sql_migration_in_progress( xformdoc.domain) if case_update.has_referrals() and not sql_migration_in_progress: logging.error( 'Form {} touching case {} in domain {} is still using referrals' .format(xformdoc.form_id, case_update.id, getattr(xformdoc, 'domain', None))) raise UsesReferrals(_('Sorry, referrals are no longer supported!')) if case_update.version and case_update.version != V2 and not sql_migration_in_progress: raise VersionNotSupported if not case_update.user_id and xformdoc.user_id: # hack for migration from V1 case blocks case_update.user_id = xformdoc.user_id if xformdoc.is_deprecated: return for action in case_update.actions: self._apply_action(case_update, action, xformdoc) # override any explicit properties from the update if case_update.user_id: self.case.modified_by = case_update.user_id modified_on = case_update.guess_modified_on() if self.case.modified_on is None or modified_on > self.case.modified_on: self.case.modified_on = modified_on # edge case if form updates case before it's been created (or creation form archived) if not self.case.opened_on: self.case.opened_on = modified_on if not self.case.opened_by: self.case.opened_by = case_update.user_id if not self.case.owner_id: self.case.owner_id = case_update.user_id
def _apply_case_update(self, case_update, xformdoc): sql_migration_in_progress = couch_sql_migration_in_progress(xformdoc.domain) if case_update.has_referrals() and not sql_migration_in_progress: logging.error('Form {} touching case {} in domain {} is still using referrals'.format( xformdoc.form_id, case_update.id, getattr(xformdoc, 'domain', None)) ) raise UsesReferrals(_('Sorry, referrals are no longer supported!')) if case_update.version and case_update.version != V2 and not sql_migration_in_progress: raise VersionNotSupported if not case_update.user_id and xformdoc.user_id: # hack for migration from V1 case blocks case_update.user_id = xformdoc.user_id if xformdoc.is_deprecated: return for action in case_update.actions: self._apply_action(case_update, action, xformdoc) # override any explicit properties from the update if case_update.user_id: self.case.modified_by = case_update.user_id modified_on = case_update.guess_modified_on() if self.case.modified_on is None or modified_on > self.case.modified_on: self.case.modified_on = modified_on # edge case if form updates case before it's been created (or creation form archived) if not self.case.opened_on: self.case.opened_on = modified_on if not self.case.opened_by: self.case.opened_by = case_update.user_id if not self.case.owner_id: self.case.owner_id = case_update.user_id
def check(cls, domain): from corehq.apps.couch_sql_migration.progress import \ couch_sql_migration_in_progress if couch_sql_migration_in_progress(domain): raise cls("couch-to-SQL migration in progress")
def is_delete_allowed(self): from corehq.apps.couch_sql_migration.progress import couch_sql_migration_in_progress return not couch_sql_migration_in_progress(self.domain)