def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write("{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write("Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table(['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'], [
(doc_type,) + stat for doc_type, stat in stats.items()
])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write(
"{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def handle(self, domain, **options):
if should_use_sql_backend(domain):
raise CommandError('It looks like {} has already been migrated.'.format(domain))
self.no_input = options.pop('no_input', False)
self.debug = options.pop('debug', False)
self.dry_run = options.pop('dry_run', False)
if self.no_input and not settings.UNIT_TESTING:
raise CommandError('no-input only allowed for unit testing')
setup_logging(options['log_dir'])
if options['MIGRATE']:
self.require_only_option('MIGRATE', options)
if options.get('run_timestamp'):
if not couch_sql_migration_in_progress(domain):
raise CommandError("Migration must be in progress if run_timestamp is passed in")
else:
set_couch_sql_migration_started(domain, self.dry_run)
do_couch_to_sql_migration(
domain,
with_progress=not self.no_input,
debug=self.debug,
run_timestamp=options.get('run_timestamp'))
has_diffs = self.print_stats(domain, short=True, diffs_only=True)
if has_diffs:
print("\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n")
if options['blow_away']:
self.require_only_option('blow_away', options)
if not self.no_input:
_confirm(
"This will delete all SQL forms and cases for the domain {}. "
"Are you sure you want to continue?".format(domain)
)
set_couch_sql_migration_not_started(domain)
blow_away_migration(domain)
if options['stats_short'] or options['stats_long']:
self.print_stats(domain, short=options['stats_short'])
if options['show_diffs']:
self.show_diffs(domain)
if options['COMMIT']:
self.require_only_option('COMMIT', options)
if not couch_sql_migration_in_progress(domain, include_dry_runs=False):
raise CommandError("cannot commit a migration that is not in state in_progress")
if not self.no_input:
_confirm(
"This will convert the domain to use the SQL backend and"
"allow new form submissions to be processed. "
"Are you sure you want to do this for domain '{}'?".format(domain)
)
set_couch_sql_migration_complete(domain)
def get_case_with_lock(self, case_id, lock=False, wrap=False):
"""
Get a case with option lock. If case not found in domains DB also check other DB
and raise IllegalCaseId if found.
:param case_id: ID of case to fetch
:param lock: Get a Redis lock for the case. Returns None if False or case not found.
:param wrap: Return wrapped case if True. (Couch only)
:return: tuple(case, lock). Either could be None
:raises: IllegalCaseId
"""
# check across Couch & SQL to ensure global uniqueness
# check this domains DB first to support existing bad data
from corehq.apps.couch_sql_migration.progress import couch_sql_migration_in_progress
case, lock = self.processor.get_case_with_lock(case_id, lock, wrap)
if case:
return case, lock
if not couch_sql_migration_in_progress(
self.domain) and not settings.ENTERPRISE_MODE:
# during migration we're copying from one DB to the other so this check will always fail
if self.other_db_processor().case_exists(case_id):
raise IllegalCaseId("Bad case id")
return case, lock
def do_MIGRATE(self, domain):
if self.finish:
assert not self.live_migrate, "--live and --finish are mutually exclusive"
elif not self.live_migrate:
status = get_couch_sql_migration_status(domain)
if status == MigrationStatus.DRY_RUN:
log.info(
"Continuing live migration. Use --finish to complete.")
self.live_migrate = True
if self.missing_docs == CACHED:
self.missing_docs = RESUME
if self.forms:
if not couch_sql_migration_in_progress(domain):
log.error("cannot migrate specific forms: migration is %s",
get_couch_sql_migration_status(domain))
sys.exit(1)
else:
set_couch_sql_migration_started(domain, self.live_migrate)
do_couch_to_sql_migration(
domain,
self.state_dir,
with_progress=not self.no_input,
live_migrate=self.live_migrate,
case_diff=self.case_diff,
rebuild_state=self.rebuild_state,
stop_on_error=self.stop_on_error,
forms=self.forms,
)
has_diffs = self.print_stats(domain, short=True, diffs_only=True)
if self.live_migrate:
print("Live migration completed.")
if has_diffs:
print("\nRun `diff` or `stats [--verbose]` for more details.\n")
sys.exit(1)
def get_case_with_lock(self, case_id, lock=False, wrap=False):
"""
Get a case with option lock. If case not found in domains DB also check other DB
and raise IllegalCaseId if found.
:param case_id: ID of case to fetch
:param lock: Get a Redis lock for the case. Returns None if False or case not found.
:param wrap: Return wrapped case if True. (Couch only)
:return: tuple(case, lock). Either could be None
:raises: IllegalCaseId
"""
# check across Couch & SQL to ensure global uniqueness
# check this domains DB first to support existing bad data
from corehq.apps.couch_sql_migration.progress import couch_sql_migration_in_progress
case, lock = self.processor.get_case_with_lock(case_id, lock, wrap)
if case:
return case, lock
if not couch_sql_migration_in_progress(self.domain) and not settings.ENTERPRISE_MODE:
# during migration we're copying from one DB to the other so this check will always fail
if self.other_db_processor().case_exists(case_id):
raise IllegalCaseId("Bad case id")
return case, lock
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return
set_couch_sql_migration_started(domain)
with SignalHandlerContext([signal.SIGTERM, signal.SIGINT],
_get_sigterm_handler(domain)):
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
else:
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green(
"Domain migrated: {}".format(domain)))
def submit_system_action(name, args, args_json, domain):
"""Submit a system action to be recorded as a form
Alias: `system_action.submit(...)`
Record a system action (un/archive form, etc.) in the stream of
submitted forms, which allows it to be reproduced during a
migration, for example.
:param name: system action name.
:param args: list of action arguments, some of which may not be
JSON-serializable.
:param args_json: a JSON-serializable object having enough
information to reconstruct `args` given the action name.
:param domain: The domain in which to perform the action.
"""
from corehq.apps.hqcase.utils import submit_case_blocks
if name not in _actions:
raise ValueError("unknown system action: {}".format(name))
if couch_sql_migration_in_progress(domain):
# record system action if migration in progress
log.debug("submit %s %s in %s", name, args_json, domain)
xml_name = escape(name)
xml_args = escape(json.dumps(args_json))
submit_case_blocks(
f"<name>{xml_name}</name><args>{xml_args}</args>",
domain,
xmlns=SYSTEM_ACTION_XMLNS,
device_id=name,
form_extras={"auth_context": SystemActionContext},
)
do_system_action(name, args)
def do_COMMIT(self, domain):
if not couch_sql_migration_in_progress(domain, include_dry_runs=False):
raise CommandError("cannot commit a migration that is not in state in_progress")
if not self.no_input:
_confirm(
"This will convert the domain to use the SQL backend and"
"allow new form submissions to be processed. "
"Are you sure you want to do this for domain '{}'?".format(domain)
)
set_couch_sql_migration_complete(domain)
def handle(self, domain, **options):
check_only = options['check']
log("Handling new reminders migration for %s, --check option is %s" %
(domain, check_only))
domain_obj = Domain.get_by_name(domain)
if self.migration_already_done(domain_obj):
return
if not check_only:
self.ensure_migration_flag_enabled(domain)
if not check_only and couch_sql_migration_in_progress(domain):
log("The Couch to SQL migration is in progress for this project, halting."
)
self.ensure_migration_flag_disabled(domain)
return
handlers = self.get_handlers_to_migrate(domain)
migrators, cannot_be_migrated = self.get_migrators(handlers)
if cannot_be_migrated:
return
log("Migration can proceed")
if check_only:
return
if not self.confirm(
"Are you sure you want to start the migration? y/n "):
log("Migrated halted")
return
self.migrate_handlers(migrators)
self.refresh_instances(domain, migrators)
log("\n")
if not self.confirm("Ok to switch on new reminders? y/n "):
log("Migrated halted")
return
self.switch_on_new_reminders(domain, migrators)
self.ensure_migration_flag_disabled(domain)
log("Migration completed.")
def handle_label(self, domain, **options):
if should_use_sql_backend(domain):
raise CommandError(
u'It looks like {} has already been migrated.'.format(domain))
self.no_input = options.pop('no_input', False)
self.debug = options.pop('debug', False)
if self.no_input and not settings.UNIT_TESTING:
raise CommandError('no-input only allowed for unit testing')
if options['MIGRATE']:
self.require_only_option('MIGRATE', options)
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain,
with_progress=not self.no_input,
debug=self.debug)
has_diffs = self.print_stats(domain, short=True, diffs_only=True)
if has_diffs:
print "\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n"
if options['blow_away']:
self.require_only_option('blow_away', options)
if not self.no_input:
_confirm(
"This will delete all SQL forms and cases for the domain {}. "
"Are you sure you want to continue?".format(domain))
set_couch_sql_migration_not_started(domain)
_blow_away_migration(domain)
if options['stats_short'] or options['stats_long']:
self.print_stats(domain, short=options['stats_short'])
if options['show_diffs']:
self.show_diffs(domain)
if options['COMMIT']:
self.require_only_option('COMMIT', options)
assert couch_sql_migration_in_progress(domain)
if not self.no_input:
_confirm(
"This will allow convert the domain to use the SQL backend and"
"allow new form submissions to be processed. "
"Are you sure you want to do this for domain '{}'?".format(
domain))
set_couch_sql_migration_complete(domain)
def handle_label(self, domain, **options):
if should_use_sql_backend(domain):
raise CommandError(u'It looks like {} has already been migrated.'.format(domain))
self.no_input = options.pop('no_input', False)
self.debug = options.pop('debug', False)
if self.no_input and not settings.UNIT_TESTING:
raise CommandError('no-input only allowed for unit testing')
if options['MIGRATE']:
self.require_only_option('MIGRATE', options)
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=not self.no_input, debug=self.debug)
has_diffs = self.print_stats(domain, short=True, diffs_only=True)
if has_diffs:
print "\nUse '--stats-short', '--stats-long', '--show-diffs' to see more info.\n"
if options['blow_away']:
self.require_only_option('blow_away', options)
if not self.no_input:
_confirm(
"This will delete all SQL forms and cases for the domain {}. "
"Are you sure you want to continue?".format(domain)
)
set_couch_sql_migration_not_started(domain)
_blow_away_migration(domain)
if options['stats_short'] or options['stats_long']:
self.print_stats(domain, short=options['stats_short'])
if options['show_diffs']:
self.show_diffs(domain)
if options['COMMIT']:
self.require_only_option('COMMIT', options)
assert couch_sql_migration_in_progress(domain)
if not self.no_input:
_confirm(
"This will allow convert the domain to use the SQL backend and"
"allow new form submissions to be processed. "
"Are you sure you want to do this for domain '{}'?".format(domain)
)
set_couch_sql_migration_complete(domain)
def _apply_case_update(self, case_update, xformdoc):
sql_migration_in_progress = couch_sql_migration_in_progress(
xformdoc.domain)
if case_update.has_referrals() and not sql_migration_in_progress:
logging.error(
'Form {} touching case {} in domain {} is still using referrals'
.format(xformdoc.form_id, case_update.id,
getattr(xformdoc, 'domain', None)))
raise UsesReferrals(_('Sorry, referrals are no longer supported!'))
if case_update.version and case_update.version != V2 and not sql_migration_in_progress:
raise VersionNotSupported
if not case_update.user_id and xformdoc.user_id:
# hack for migration from V1 case blocks
case_update.user_id = xformdoc.user_id
if xformdoc.is_deprecated:
return
for action in case_update.actions:
self._apply_action(case_update, action, xformdoc)
# override any explicit properties from the update
if case_update.user_id:
self.case.modified_by = case_update.user_id
modified_on = case_update.guess_modified_on()
if self.case.modified_on is None or modified_on > self.case.modified_on:
self.case.modified_on = modified_on
# edge case if form updates case before it's been created (or creation form archived)
if not self.case.opened_on:
self.case.opened_on = modified_on
if not self.case.opened_by:
self.case.opened_by = case_update.user_id
if not self.case.owner_id:
self.case.owner_id = case_update.user_id
def _apply_case_update(self, case_update, xformdoc):
sql_migration_in_progress = couch_sql_migration_in_progress(xformdoc.domain)
if case_update.has_referrals() and not sql_migration_in_progress:
logging.error('Form {} touching case {} in domain {} is still using referrals'.format(
xformdoc.form_id, case_update.id, getattr(xformdoc, 'domain', None))
)
raise UsesReferrals(_('Sorry, referrals are no longer supported!'))
if case_update.version and case_update.version != V2 and not sql_migration_in_progress:
raise VersionNotSupported
if not case_update.user_id and xformdoc.user_id:
# hack for migration from V1 case blocks
case_update.user_id = xformdoc.user_id
if xformdoc.is_deprecated:
return
for action in case_update.actions:
self._apply_action(case_update, action, xformdoc)
# override any explicit properties from the update
if case_update.user_id:
self.case.modified_by = case_update.user_id
modified_on = case_update.guess_modified_on()
if self.case.modified_on is None or modified_on > self.case.modified_on:
self.case.modified_on = modified_on
# edge case if form updates case before it's been created (or creation form archived)
if not self.case.opened_on:
self.case.opened_on = modified_on
if not self.case.opened_by:
self.case.opened_by = case_update.user_id
if not self.case.owner_id:
self.case.owner_id = case_update.user_id
def check(cls, domain):
from corehq.apps.couch_sql_migration.progress import \
couch_sql_migration_in_progress
if couch_sql_migration_in_progress(domain):
raise cls("couch-to-SQL migration in progress")
def is_delete_allowed(self):
from corehq.apps.couch_sql_migration.progress import couch_sql_migration_in_progress
return not couch_sql_migration_in_progress(self.domain)
