def test_change_provider_empty(self):
provider = CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[],
doc_types=all_known_formlike_doc_types())
self.assertEqual([],
[change for change in provider.iter_all_changes()])
def test_change_provider_empty(self):
provider = CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[],
doc_types=all_known_formlike_doc_types()
)
self.assertEqual([], [change for change in provider.iter_all_changes()])
def test_change_provider(self):
provider = CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=self.domains,
doc_types=all_known_formlike_doc_types())
doc_ids = {change.id for change in provider.iter_all_changes()}
self.assertEqual(doc_ids,
set(itertools.chain(*self.form_ids.values())))
def test_change_provider(self):
provider = CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=self.domains,
doc_types=all_known_formlike_doc_types()
)
doc_ids = {change.id for change in provider.iter_all_changes()}
self.assertEqual(doc_ids, set(itertools.chain(*list(self.form_ids.values()))))
def _get_unprocessed_form_iterator(domain):
return CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[domain],
doc_types=UNPROCESSED_DOC_TYPES,
event_handler=MigrationPaginationEventHandler(domain),
)
def _get_main_form_iterator(domain):
return CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[domain],
doc_types=['XFormInstance'],
event_handler=MigrationPaginationEventHandler(domain),
)
def _get_unprocessed_form_iterator(domain):
return CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[domain],
doc_types=UNPROCESSED_DOC_TYPES,
event_handler=ReindexEventHandler(u'couch to sql migrator ({} unprocessed forms)'.format(domain)),
)
def _get_main_form_iterator(domain):
return CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[domain],
doc_types=['XFormInstance'],
event_handler=ReindexEventHandler(u'couch to sql migrator ({})'.format(domain)),
)
def _iter_changes(domain, doc_types, **kw):
return CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[domain],
doc_types=doc_types,
event_handler=MigrationPaginationEventHandler(domain),
).iter_all_changes(**kw)
def _get_case_iterator(domain, doc_types=None):
doc_types = doc_types or CASE_DOC_TYPES
return CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[domain],
doc_types=doc_types,
event_handler=MigrationPaginationEventHandler(domain),
)
def _get_case_iterator(domain, doc_types=None):
doc_types = doc_types or CASE_DOC_TYPES
return CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[domain],
doc_types=doc_types,
event_handler=ReindexEventHandler(u'couch to sql migrator ({})'.format(domain)),
)
def get_domain_form_change_provider(domains):
sql_domains = {
domain
for domain in domains if should_use_sql_backend(domain)
}
couch_domains = set(domains) - sql_domains
return CompositeChangeProvider([
SqlDomainXFormChangeProvider(sql_domains),
CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=couch_domains,
doc_types=all_known_formlike_doc_types()),
])
def handle(self, *args, **options):
if len(args) < 1:
raise CommandError('Usage is ptop_reindexer_fluff %s' % self.args)
fluff_configs = {
config.name: config
for config in get_fluff_pillow_configs()
}
pillow_name = args[0]
if pillow_name not in fluff_configs:
raise CommandError(
'Unrecognised fluff pillow: "{}". Options are:\n\t{}'.format(
pillow_name, '\n\t'.join(fluff_configs)))
pillow_getter = get_pillow_by_name(pillow_name, instantiate=False)
pillow = pillow_getter(delete_filtered=True)
if len(args) == 1:
domains = pillow.domains
else:
domains = args[1:]
domains_not_in_pillow = set(domains) - set(pillow.domains)
if domains_not_in_pillow:
bad_domains = ', '.join(domains_not_in_pillow)
available_domains = ', '.join(pillow.domains)
raise CommandError(
"The following domains aren't for this pillow: {}.\nAvailable domains are: {}"
.format(bad_domains, available_domains))
if pillow.kafka_topic in (topics.CASE, topics.FORM):
couch_db = couch_config.get_db(None)
elif pillow.kafka_topic == topics.COMMCARE_USER:
couch_db = couch_config.get_db(settings.NEW_USERS_GROUPS_DB)
else:
raise CommandError('Reindexer not configured for topic: {}'.format(
pillow.kafka_topic))
change_provider = CouchDomainDocTypeChangeProvider(
couch_db=couch_db,
domains=domains,
doc_types=[pillow.doc_type],
event_handler=ReindexEventHandler(pillow_name),
)
PillowChangeProviderReindexer(pillow, change_provider).reindex()
def _get_unprocessed_form_iterator(domain):
return CouchDomainDocTypeChangeProvider(
couch_db=XFormInstance.get_db(),
domains=[domain],
doc_types=[
'XFormArchived',
'XFormError',
'XFormDeprecated',
'XFormDuplicate',
# todo: need to figure out which of these we plan on supporting
'XFormInstance-Deleted',
'HQSubmission',
'SubmissionErrorLog',
],
event_handler=ReindexEventHandler(
u'couch to sql migrator ({} unprocessed forms)'.format(domain)),
)
