def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write(
"{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def handle(self, domain, doc_type, **options):
csv = options.get('csv')
handlers = {
'CommCareCase': _compare_cases,
'CommCareCase-Deleted': _compare_cases,
'CommCareUser': _compare_users,
'CommCareUser-Deleted': _compare_users,
'WebUser': _compare_users,
}
handlers.update(
{doc_type: _compare_xforms
for doc_type in doc_types()})
try:
primary_only, es_only = handlers[doc_type](domain, doc_type)
except KeyError:
raise CommandError('Unsupported doc type. Use on of: {}'.format(
', '.join(handlers)))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([50, 50])
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(['Only in Primary', 'Only in ES'],
izip_longest(primary_only, es_only, fillvalue=''))
def print_stats(self):
stats = UnfinishedSubmissionStub.objects.all().values('domain')\
.annotate(count=Count('domain')).order_by('count')
writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10]))
writer.write_table(['Domain', '# Forms'], [
(stat['domain'], stat['count']) for stat in stats
])
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return
set_couch_sql_migration_started(domain)
with SignalHandlerContext([signal.SIGTERM, signal.SIGINT],
_get_sigterm_handler(domain)):
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
else:
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green(
"Domain migrated: {}".format(domain)))
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
log.error("{} already on the SQL backend\n".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
log.error("{} migration is already in progress\n".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
lines = ["Migration has diffs: {}".format(domain)]
class stream:
write = lines.append
writer = SimpleTableWriter(stream, TableRowFormatter([30, 10, 10, 10, 10]))
writer.write_table(
['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'],
[(doc_type,) + stat for doc_type, stat in stats.items()],
)
log.error("\n".join(lines))
self.abort(domain)
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
log.info("Domain migrated: {}\n".format(domain))
return True, None
def print_stats(self):
stats = UnfinishedSubmissionStub.objects.all().values('domain')\
.annotate(count=Count('domain')).order_by('count')
writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10]))
writer.write_table(['Domain', '# Forms'], [
(stat['domain'], stat['count']) for stat in stats
])
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write("{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write("Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table(['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'], [
(doc_type,) + stat for doc_type, stat in stats.items()
])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def _write_info(title, info, row_formatter, out=None):
out = out or sys.stdout
writer = SimpleTableWriter(out, row_formatter)
out.write('\n{}\n'.format(title))
out.write('{}\n'.format(row_formatter.format_row(['Total', info.total, ''])))
out.write('\nBy DB\n'.format(info.total))
writer.write_table(['DB', '# Docs', '%'], info.by_db_table)
out.write('\nBy Host\n'.format(info.total))
writer.write_table(['Host', '# Docs', '%'], info.by_host_table)
def handle(self, domain, doc_type, **options):
csv = options.get('csv')
startdate = options.get('start')
enddate = options.get('end')
form_doc_types = doc_types()
if startdate or enddate:
if doc_type in CASE_DOC_TYPES or doc_type in form_doc_types:
if not should_use_sql_backend(domain):
raise CommandError(
"Date filtering not supported for Couch domains")
if startdate and enddate and enddate <= startdate:
raise CommandError("enddate must be after startdate")
handlers = {
'CommCareCase': compare_cases,
'CommCareCase-Deleted': compare_cases,
'CommCareUser': _compare_users,
'CommCareUser-Deleted': _compare_users,
'WebUser': _compare_users,
}
handlers.update(
{doc_type: compare_xforms
for doc_type in form_doc_types})
try:
primary_count, es_count, primary_only, es_only = handlers[
doc_type](domain, doc_type, startdate, enddate)
except KeyError:
raise CommandError('Unsupported doc type. Use on of: {}'.format(
', '.join(handlers)))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([50, 50])
date_range_output = ''
if startdate or enddate:
end = (enddate or datetime.utcnow().date()).strftime(DATE_FORMAT)
start = startdate.strftime(DATE_FORMAT)
date_range_output = ' (Between {} and {})'.format(start, end)
print("\nDoc ID analysis for {}{}\n".format(doc_type,
date_range_output))
print("Primary Count: {}".format(primary_count))
print("ES Count: {}\n".format(es_count))
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(['Only in Primary', 'Only in ES'],
zip_longest(primary_only, es_only, fillvalue=''))
def handle(self, domain, doc_type, **options):
csv = options.get('csv')
startdate = options.get('start')
enddate = options.get('end')
form_doc_types = doc_types()
if startdate or enddate:
if doc_type in CASE_DOC_TYPES or doc_type in form_doc_types:
if not should_use_sql_backend(domain):
raise CommandError("Date filtering not supported for Couch domains")
if startdate and enddate and enddate <= startdate:
raise CommandError("enddate must be after startdate")
handlers = {
'CommCareCase': compare_cases,
'CommCareCase-Deleted': compare_cases,
'CommCareUser': _compare_users,
'CommCareUser-Deleted': _compare_users,
'WebUser': _compare_users,
}
handlers.update({doc_type: compare_xforms for doc_type in form_doc_types})
try:
primary_count, es_count, primary_only, es_only = handlers[doc_type](domain, doc_type, startdate, enddate)
except KeyError:
raise CommandError('Unsupported doc type. Use on of: {}'.format(', '.join(handlers)))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([50, 50])
date_range_output = ''
if startdate or enddate:
end = (enddate or datetime.utcnow().date()).strftime(DATE_FORMAT)
start = startdate.strftime(DATE_FORMAT)
date_range_output = ' (Between {} and {})'.format(start, end)
print("\nDoc ID analysis for {}{}\n".format(doc_type, date_range_output))
print("Primary Count: {}".format(primary_count))
print("ES Count: {}\n".format(es_count))
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(
['Only in Primary', 'Only in ES'],
zip_longest(primary_only, es_only, fillvalue='')
)
def format_diff_stats(stats, header=None):
lines = []
if stats:
if header:
lines.append(header)
class stream:
write = lines.append
writer = SimpleTableWriter(stream, TableRowFormatter([30, 10, 10, 10]))
writer.write_table(
['Doc Type', '# Couch', '# SQL', '# Docs with Diffs'],
[(doc_type,) + stat for doc_type, stat in stats.items()],
)
return "\n".join(lines)
def format_diff_stats(stats, header=None):
lines = []
if stats:
if header:
lines.append(header)
class stream:
write = lines.append
writer = SimpleTableWriter(stream, TableRowFormatter([30, 7, 7, 7, 7]))
writer.write_table(
['Doc Type', 'Docs', 'Diffs', "Missing", "Changes"],
[(doc_type, ) + stat.columns for doc_type, stat in stats.items()],
)
return "\n".join(lines)
def handle(self, pillow_name, process_count, **options):
all_pillows = get_all_pillow_instances()
pillows = [
pillow for pillow in all_pillows if pillow.pillow_id == pillow_name
]
if not pillows:
all_names = '\n\t'.join(
[pillow.pillow_id for pillow in all_pillows])
raise CommandError(
f'No pillow found: {pillow_name}.\n\t{all_names}')
pillow = pillows[0]
change_feed = pillow.get_change_feed()
if not isinstance(change_feed, KafkaChangeFeed):
raise CommandError(f"Pillow '{pillow_name}' is not a Kafka pillow")
topic_partitions = []
for topic in change_feed.topics:
for partition in change_feed.consumer.partitions_for_topic(topic):
topic_partitions.append(TopicPartition(topic, partition))
topic_partitions.sort()
process_assignments = [
topic_partitions[num::process_count]
for num in range(process_count)
]
if options['csv']:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([len(pillow_name) + 4, 10])
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(['Pillow Process', 'Kafka Topics'], [[
f'{pillow_name}:{process_num}', ', '.join(
[f'{tp.topic}-{tp.partition}' for tp in assignments])
] for process_num, assignments in enumerate(process_assignments)])
