def print_stats(self):
stats = UnfinishedSubmissionStub.objects.all().values('domain')\
.annotate(count=Count('domain')).order_by('count')
writer = SimpleTableWriter(self.stdout, TableRowFormatter([50, 10]))
writer.write_table(['Domain', '# Forms'], [
(stat['domain'], stat['count']) for stat in stats
])
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
self.stderr.write(
"{} migration is already in progress".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green("Domain migrated: {}".format(domain)))
return True, None
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
log.error("{} already on the SQL backend\n".format(domain))
return True, None
if couch_sql_migration_in_progress(domain, include_dry_runs=True):
log.error("{} migration is already in progress\n".format(domain))
return False, "in progress"
set_couch_sql_migration_started(domain)
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
lines = ["Migration has diffs: {}".format(domain)]
class stream:
write = lines.append
writer = SimpleTableWriter(stream, TableRowFormatter([30, 10, 10, 10, 10]))
writer.write_table(
['Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'],
[(doc_type,) + stat for doc_type, stat in stats.items()],
)
log.error("\n".join(lines))
self.abort(domain)
return False, "has diffs"
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
log.info("Domain migrated: {}\n".format(domain))
return True, None
def handle(self, domain, doc_type, **options):
csv = options.get('csv')
handlers = {
'CommCareCase': _compare_cases,
'CommCareCase-Deleted': _compare_cases,
'CommCareUser': _compare_users,
'CommCareUser-Deleted': _compare_users,
'WebUser': _compare_users,
}
handlers.update(
{doc_type: _compare_xforms
for doc_type in doc_types()})
try:
primary_only, es_only = handlers[doc_type](domain, doc_type)
except KeyError:
raise CommandError('Unsupported doc type. Use on of: {}'.format(
', '.join(handlers)))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([50, 50])
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(['Only in Primary', 'Only in ES'],
izip_longest(primary_only, es_only, fillvalue=''))
def handle(self, domain, **options):
csv = options.get('csv')
couch_counts = map_counter_doc_types(_get_couchdb_counts(domain))
sql_counts = map_counter_doc_types(_get_sql_counts(domain))
es_counts = map_counter_doc_types(get_es_counts_by_doc_type(domain))
all_doc_types = set(couch_counts) | set(sql_counts) | set(es_counts)
output_rows = []
for doc_type in sorted(all_doc_types, key=lambda d: d.lower()):
couch = couch_counts.get(doc_type, '')
sql = sql_counts.get(doc_type, '')
es = es_counts.get(doc_type, '')
output_rows.append((doc_type, couch, sql, es))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter(
[50, 20, 20, 20],
_get_row_color
)
SimpleTableWriter(self.stdout, row_formatter).write_table(
['Doc Type', 'Couch', 'SQL', 'ES'], output_rows
)
def migrate_domain(self, domain):
if should_use_sql_backend(domain):
self.stderr.write("{} already on the SQL backend".format(domain))
return
set_couch_sql_migration_started(domain)
with SignalHandlerContext([signal.SIGTERM, signal.SIGINT],
_get_sigterm_handler(domain)):
do_couch_to_sql_migration(domain, with_progress=False, debug=False)
stats = self.get_diff_stats(domain)
if stats:
self.stderr.write(
"Migration has diffs, aborting for domain {}".format(domain))
self.abort(domain)
writer = SimpleTableWriter(self.stdout,
TableRowFormatter([50, 10, 10, 10, 10]))
writer.write_table([
'Doc Type', '# Couch', '# SQL', '# Diffs', '# Docs with Diffs'
], [(doc_type, ) + stat for doc_type, stat in stats.items()])
else:
assert couch_sql_migration_in_progress(domain)
set_couch_sql_migration_complete(domain)
self.stdout.write(shell_green(
"Domain migrated: {}".format(domain)))
def _print_table(self, headers, rows):
if self.csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([20] * len(headers), )
SimpleTableWriter(self.stdout,
row_formatter).write_table(headers, rows)
self.stdout.write('')
def handle(self, domain, doc_type, **options):
csv = options.get('csv')
startdate = options.get('start')
enddate = options.get('end')
form_doc_types = doc_types()
if startdate or enddate:
if doc_type in CASE_DOC_TYPES or doc_type in form_doc_types:
if not should_use_sql_backend(domain):
raise CommandError(
"Date filtering not supported for Couch domains")
if startdate and enddate and enddate <= startdate:
raise CommandError("enddate must be after startdate")
handlers = {
'CommCareCase': compare_cases,
'CommCareCase-Deleted': compare_cases,
'CommCareUser': _compare_users,
'CommCareUser-Deleted': _compare_users,
'WebUser': _compare_users,
}
handlers.update(
{doc_type: compare_xforms
for doc_type in form_doc_types})
try:
primary_count, es_count, primary_only, es_only = handlers[
doc_type](domain, doc_type, startdate, enddate)
except KeyError:
raise CommandError('Unsupported doc type. Use on of: {}'.format(
', '.join(handlers)))
if csv:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([50, 50])
date_range_output = ''
if startdate or enddate:
end = (enddate or datetime.utcnow().date()).strftime(DATE_FORMAT)
start = startdate.strftime(DATE_FORMAT)
date_range_output = ' (Between {} and {})'.format(start, end)
print("\nDoc ID analysis for {}{}\n".format(doc_type,
date_range_output))
print("Primary Count: {}".format(primary_count))
print("ES Count: {}\n".format(es_count))
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(['Only in Primary', 'Only in ES'],
zip_longest(primary_only, es_only, fillvalue=''))
def handle(self, **options):
csv_mode = options['csv']
form_info = _get_counts(FormReindexAccessor())
case_info = _get_counts(CaseReindexAccessor())
if csv_mode:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([20, 20, 10])
_write_info('FORMS', form_info, row_formatter, self.stdout)
_write_info('CASES', case_info, row_formatter, self.stdout)
def format_diff_stats(stats, header=None):
lines = []
if stats:
if header:
lines.append(header)
class stream:
write = lines.append
writer = SimpleTableWriter(stream, TableRowFormatter([30, 10, 10, 10]))
writer.write_table(
['Doc Type', '# Couch', '# SQL', '# Docs with Diffs'],
[(doc_type,) + stat for doc_type, stat in stats.items()],
)
return "\n".join(lines)
def format_diff_stats(stats, header=None):
lines = []
if stats:
if header:
lines.append(header)
class stream:
write = lines.append
writer = SimpleTableWriter(stream, TableRowFormatter([30, 7, 7, 7, 7]))
writer.write_table(
['Doc Type', 'Docs', 'Diffs', "Missing", "Changes"],
[(doc_type, ) + stat.columns for doc_type, stat in stats.items()],
)
return "\n".join(lines)
def handle(self, domains, **options):
csv = options.get('csv')
if csv:
row_formatter = CSVRowFormatter()
else:
dom_len = max(len(domain) for domain in domains) + 2
row_formatter = TableRowFormatter([dom_len, 7, 50, 15, 15],
_get_row_color)
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_headers(
['Domain', 'Backend', 'Doc Type', 'Primary', 'ES'])
for domain in domains:
rows = get_doc_counts_for_domain(domain)
writer.write_rows(((domain, 'sql') + row for row in rows))
def handle(self, pillow_name, process_count, **options):
all_pillows = get_all_pillow_instances()
pillows = [
pillow for pillow in all_pillows if pillow.pillow_id == pillow_name
]
if not pillows:
all_names = '\n\t'.join(
[pillow.pillow_id for pillow in all_pillows])
raise CommandError(
f'No pillow found: {pillow_name}.\n\t{all_names}')
pillow = pillows[0]
change_feed = pillow.get_change_feed()
if not isinstance(change_feed, KafkaChangeFeed):
raise CommandError(f"Pillow '{pillow_name}' is not a Kafka pillow")
topic_partitions = []
for topic in change_feed.topics:
for partition in change_feed.consumer.partitions_for_topic(topic):
topic_partitions.append(TopicPartition(topic, partition))
topic_partitions.sort()
process_assignments = [
topic_partitions[num::process_count]
for num in range(process_count)
]
if options['csv']:
row_formatter = CSVRowFormatter()
else:
row_formatter = TableRowFormatter([len(pillow_name) + 4, 10])
writer = SimpleTableWriter(self.stdout, row_formatter)
writer.write_table(['Pillow Process', 'Kafka Topics'], [[
f'{pillow_name}:{process_num}', ', '.join(
[f'{tp.topic}-{tp.partition}' for tp in assignments])
] for process_num, assignments in enumerate(process_assignments)])