def get_document_store_for_doc_type(domain,
doc_type,
case_type_or_xmlns=None,
load_source="unknown"):
"""Only applies to documents that have a document type:
* forms
* cases
* locations
* leddgers (V2 only)
* all couch models
"""
from corehq.apps.change_feed import document_types
if doc_type in all_known_formlike_doc_types():
store = FormDocumentStore(domain, xmlns=case_type_or_xmlns)
load_counter = form_load_counter
elif doc_type in document_types.CASE_DOC_TYPES:
store = CaseDocumentStore(domain, case_type=case_type_or_xmlns)
load_counter = case_load_counter
elif doc_type == LOCATION_DOC_TYPE:
return LocationDocumentStore(domain)
elif doc_type == topics.LEDGER:
return LedgerV2DocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type)
track_load = load_counter(load_source, domain)
return DocStoreLoadTracker(store, track_load)
def get_document_store_for_doc_type(domain, doc_type, case_type_or_xmlns=None, load_source="unknown"):
"""Only applies to documents that have a document type:
* forms
* cases
* locations
* all couch models
"""
from corehq.apps.change_feed import document_types
if doc_type in all_known_formlike_doc_types():
store = ReadonlyFormDocumentStore(domain, xmlns=case_type_or_xmlns)
load_counter = form_load_counter
elif doc_type in document_types.CASE_DOC_TYPES:
store = ReadonlyCaseDocumentStore(domain, case_type=case_type_or_xmlns)
load_counter = case_load_counter
elif doc_type == LOCATION_DOC_TYPE:
return ReadonlyLocationDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(
couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type
)
track_load = load_counter(load_source, domain)
return DocStoreLoadTracker(store, track_load)
def configure(self, doc_id_expression, value_expression):
non_couch_doc_types = (LOCATION_DOC_TYPE,)
if (self.related_doc_type not in non_couch_doc_types
and get_db_by_doc_type(self.related_doc_type) is None):
raise BadSpecError('Cannot determine database for document type {}!'.format(self.related_doc_type))
self._doc_id_expression = doc_id_expression
self._value_expression = value_expression
def configure(self, doc_id_expression, value_expression):
non_couch_doc_types = (LOCATION_DOC_TYPE,)
if (self.related_doc_type not in non_couch_doc_types
and get_db_by_doc_type(self.related_doc_type) is None):
raise BadSpecError('Cannot determine database for document type {}!'.format(self.related_doc_type))
self._doc_id_expression = doc_id_expression
self._value_expression = value_expression
def configure(self, doc_id_expression, value_expression):
if get_db_by_doc_type(self.related_doc_type) is None:
raise BadSpecError(u'Cannot determine database for document type {}!'.format(self.related_doc_type))
self._doc_id_expression = doc_id_expression
self._value_expression = value_expression
# used in caching
self._vary_on = json.dumps(self.value_expression, sort_keys=True)
def _get_db_for_doc_type(self, doc_type):
if doc_type not in self._dbs:
couch_db = get_db_by_doc_type(doc_type)
callback = LoaderCallback(self.success_counter, self.stdout)
db = IterDB(couch_db, new_edits=False, callback=callback)
db.__enter__()
self._dbs[doc_type] = db
return self._dbs[doc_type]
def _get_db_for_doc_type(self, doc_type):
if doc_type not in self._dbs:
couch_db = get_db_by_doc_type(doc_type)
callback = LoaderCallback(self.success_counter, self.stdout)
db = IterDB(couch_db, new_edits=False, callback=callback)
db.__enter__()
self._dbs[doc_type] = db
return self._dbs[doc_type]
def configure(self, doc_id_expression, value_expression):
if get_db_by_doc_type(self.related_doc_type) is None:
raise BadSpecError(
u'Cannot determine database for document type {}!'.format(
self.related_doc_type))
self._doc_id_expression = doc_id_expression
self._value_expression = value_expression
# used in caching
self._vary_on = json.dumps(self.value_expression, sort_keys=True)
def _get_db_for_doc_type(self, doc_type):
if doc_type not in self._dbs:
couch_db = get_db_by_doc_type(doc_type)
if couch_db is None:
raise DocumentClassNotFound('No Document class with name "{}" could be found.'.format(doc_type))
callback = LoaderCallback(self.success_counter, self.stdout)
db = IterDB(couch_db, new_edits=False, callback=callback)
db.__enter__()
self._dbs[doc_type] = db
return self._dbs[doc_type]
def handle(self, doc_types, *args, **options):
input = raw_input(
"\n".join(
[
"\n\nReally delete documents of the following types: {}?",
"This operation is not reversible. Enter a number N to delete the first "
'N found, or type "delete all" to delete everything.',
"",
]
).format(doc_types)
)
if input == "delete all":
remaining = None
else:
try:
remaining = int(input)
except ValueError:
print "aborting"
sys.exit()
doc_types = doc_types.split(",")
deleted = 0
# unfortunately the only couch view we have for this needs to go by domain
# will be a bit slow
domain_names = Domain.get_all_names()
for doc_type in doc_types:
db = get_db_by_doc_type(doc_type)
if not db:
print "Cannot find db for {}, skipping".format(doc_type)
continue
for domain in domain_names:
docs = [
row["doc"]
for row in db.view(
"by_domain_doc_type_date/view",
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=True,
)
][:remaining]
if docs:
count = len(docs)
print "deleting {} {}s from {}".format(count, doc_type, domain)
db.delete_docs(docs)
deleted += count
if remaining is not None:
remaining -= count
if remaining <= 0:
return
print "successfully deleted {} documents".format(deleted)
def get_document_store(domain, doc_type):
use_sql = should_use_sql_backend(domain)
if use_sql and doc_type == 'XFormInstance':
return ReadonlyFormDocumentStore(domain)
elif use_sql and doc_type == 'CommCareCase':
return ReadonlyCaseDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type)
def get_value(self, doc_id, context):
try:
doc = get_db_by_doc_type(self.related_doc_type).get(doc_id)
# ensure no cross-domain lookups of different documents
assert context.root_doc['domain']
if context.root_doc['domain'] != doc.get('domain'):
return None
# explicitly use a new evaluation context since this is a new document
return self._value_expression(doc, EvaluationContext(doc, 0))
except ResourceNotFound:
return None
def get_document_store(domain, doc_type, case_type_or_xmlns=None):
if doc_type == 'XFormInstance':
return ReadonlyFormDocumentStore(domain, xmlns=case_type_or_xmlns)
elif doc_type == 'CommCareCase':
return ReadonlyCaseDocumentStore(domain, case_type=case_type_or_xmlns)
elif doc_type == LOCATION_DOC_TYPE:
return ReadonlyLocationDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type)
def get_doc_ids_in_domain_by_type(domain, doc_type, database=None):
"""
Given a domain and doc type, get all docs matching that domain and type
"""
if not database:
database = get_db_by_doc_type(doc_type)
return [row['id'] for row in database.view('by_domain_doc_type_date/view',
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=False,
)]
def get_doc_ids_in_domain_by_type(domain, doc_type, database=None):
"""
Given a domain and doc type, get all docs matching that domain and type
"""
if not database:
database = get_db_by_doc_type(doc_type)
return [row['id'] for row in database.view('by_domain_doc_type_date/view',
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=False,
)]
def get_document_store(domain, doc_type):
use_sql = should_use_sql_backend(domain)
if use_sql and doc_type == 'XFormInstance':
return ReadonlyFormDocumentStore(domain)
elif use_sql and doc_type == 'CommCareCase':
return ReadonlyCaseDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(
couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type
)
def handle(self, doc_types, *args, **options):
user_input = input('\n'.join([
'\n\nReally delete documents of the following types: {}?',
'This operation is not reversible. Enter a number N to delete the first '
'N found, or type "delete all" to delete everything.',
'',
]).format(doc_types))
if user_input == 'delete all':
remaining = None
else:
try:
remaining = int(user_input)
except ValueError:
print('aborting')
sys.exit()
doc_types = doc_types.split(',')
deleted = 0
# unfortunately the only couch view we have for this needs to go by domain
# will be a bit slow
domain_names = Domain.get_all_names()
for doc_type in doc_types:
db = get_db_by_doc_type(doc_type)
if not db:
print("Cannot find db for {}, skipping".format(doc_type))
continue
for domain in domain_names:
docs = [
row['doc'] for row in db.view(
'by_domain_doc_type_date/view',
startkey=[domain, doc_type],
endkey=[domain, doc_type, {}],
reduce=False,
include_docs=True,
)
][:remaining]
if docs:
count = len(docs)
print('deleting {} {}s from {}'.format(
count, doc_type, domain))
db.delete_docs(docs)
deleted += count
if remaining is not None:
remaining -= count
if remaining <= 0:
return
print('successfully deleted {} documents'.format(deleted))
def configure(self, doc_id_expression, value_expression):
non_couch_doc_types = {
CommCareCase.DOC_TYPE,
LOCATION_DOC_TYPE,
XFormInstance.STATE_TO_DOC_TYPE[XFormInstance.NORMAL],
}
if (self.related_doc_type not in non_couch_doc_types
and get_db_by_doc_type(self.related_doc_type) is None):
raise BadSpecError(
'Cannot determine database for document type {}!'.format(
self.related_doc_type))
self._doc_id_expression = doc_id_expression
self._value_expression = value_expression
def iterate_doc_ids_in_domain_by_type(domain, doc_type, chunk_size=10000, database=None):
if not database:
database = get_db_by_doc_type(doc_type)
view_kwargs = {
'reduce': False,
'startkey': [domain, doc_type],
'endkey': [domain, doc_type, {}],
'include_docs': False
}
for doc in paginate_view(
database,
'by_domain_doc_type_date/view',
chunk_size,
**view_kwargs):
yield doc['id']
def get_document_store_for_doc_type(domain, doc_type, case_type_or_xmlns=None):
"""Only applies to documents that have a document type:
* forms
* cases
* locations
* all couch models
"""
from corehq.apps.change_feed import document_types
if doc_type in all_known_formlike_doc_types():
return ReadonlyFormDocumentStore(domain, xmlns=case_type_or_xmlns)
elif doc_type in document_types.CASE_DOC_TYPES:
return ReadonlyCaseDocumentStore(domain, case_type=case_type_or_xmlns)
elif doc_type == LOCATION_DOC_TYPE:
return ReadonlyLocationDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type)
def iterate_doc_ids_in_domain_by_type(domain,
doc_type,
chunk_size=10000,
database=None,
startkey=None,
startkey_docid=None):
if not database:
database = get_db_by_doc_type(doc_type)
view_kwargs = {
'reduce': False,
'startkey': startkey if startkey else [domain, doc_type],
'endkey': [domain, doc_type, {}],
'include_docs': False
}
if startkey_docid:
view_kwargs.update({'startkey_docid': startkey_docid})
for doc in paginate_view(database, 'by_domain_doc_type_date/view',
chunk_size, **view_kwargs):
yield doc['id']
def get_document_store_for_doc_type(domain, doc_type, case_type_or_xmlns=None):
"""Only applies to documents that have a document type:
* forms
* cases
* locations
* all couch models
"""
from corehq.apps.change_feed import document_types
if doc_type in all_known_formlike_doc_types():
return ReadonlyFormDocumentStore(domain, xmlns=case_type_or_xmlns)
elif doc_type in document_types.CASE_DOC_TYPES:
return ReadonlyCaseDocumentStore(domain, case_type=case_type_or_xmlns)
elif doc_type == LOCATION_DOC_TYPE:
return ReadonlyLocationDocumentStore(domain)
else:
# all other types still live in couchdb
return CouchDocumentStore(
couch_db=get_db_by_doc_type(doc_type),
domain=domain,
doc_type=doc_type
)
