def handle(self, *args, **options):
if len(args) < 2:
raise CommandError('Usage is copy_case, %s' % self.args)
sourcedb = Database(args[0])
case_id = args[1]
domain = args[2] if len(args) > 2 else None
print 'getting case'
case = CommCareCase.wrap(sourcedb.get(case_id))
if domain is not None:
case.domain = domain
case.save(force_update=True)
print 'copying %s xforms' % len(case.xform_ids)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
return XFormInstance.wrap(doc)
xforms = sourcedb.all_docs(
keys=case.xform_ids,
include_docs=True,
wrapper=form_wrapper,
).all()
for form in xforms:
if domain is not None:
form.domain = domain
form.save(force_update=True)
print 'saved %s' % form._id
def handle(self, *args, **options):
raise CommandError(
'copy_group_data is currently broken. '
'Ask Danny or Ethan to fix it along the lines of '
'https://github.com/dimagi/commcare-hq/pull/9180/files#diff-9d976dc051a36a028c6604581dfbce5dR95'
)
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains',
key=group.domain,
include_docs=True,
reduce=False,
limit=1).one()['doc'])
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
print 'getting case ids'
with OverrideDB(CommCareCase, sourcedb):
case_ids = get_case_ids_in_domain_by_owner(domain.name,
owner_id__in=owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [
CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)
]
for case in cases:
def handle(self, *args, **options):
if len(args) < 2 or len(args) > 3:
raise CommandError('Usage is copy_doc %s' % self.args)
sourcedb = Database(args[0])
app_id = args[1]
domain = args[2] if len(args) == 3 else None
app_json = sourcedb.get(app_id)
if domain:
app_json['domain'] = domain
dt = DocumentTransform(app_json, sourcedb)
save(dt, get_db())
def handle(self, sourcedb, doc_ids_or_file, domain, **options):
sourcedb = Database(sourcedb)
if os.path.isfile(doc_ids_or_file):
with open(doc_ids_or_file) as f:
doc_ids = f.read().splitlines()
else:
doc_ids = doc_ids_or_file.split(',')
print("Starting copy of {} docs".format(len(doc_ids)))
for doc_id in doc_ids:
print('Copying doc: {}'.format(doc_id))
doc_json = sourcedb.get(doc_id)
if domain:
doc_json['domain'] = domain
dt = DocumentTransform(doc_json, sourcedb)
save(dt, get_db())
def _lookup_id_in_couch(doc_id):
db_urls = [settings.COUCH_DATABASE] + settings.EXTRA_COUCHDB_DATABASES.values()
for url in db_urls:
db = Database(url)
try:
doc = db.get(doc_id)
except ResourceNotFound:
pass
else:
return {
"doc": json.dumps(doc, indent=4, sort_keys=True),
"doc_id": doc_id,
"doc_type": doc.get('doc_type', 'Unknown'),
"dbname": db.dbname,
}
return {
"doc": "NOT FOUND",
"doc_id": doc_id,
}
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains', key=group.domain, include_docs=True,
reduce=False, limit=1).one()['doc']
)
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
print 'getting case ids'
with OverrideDB(CommCareCase, sourcedb):
case_ids = get_case_ids_in_domain_by_owner(
domain.name, owner_id__in=owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
for case in cases:
def doc_in_es(request):
doc_id = request.GET.get("id")
if not doc_id:
return render(request, "hqadmin/doc_in_es.html", {})
couch_doc = {}
db_urls = [settings.COUCH_DATABASE] + settings.EXTRA_COUCHDB_DATABASES.values()
for url in db_urls:
try:
couch_doc = Database(url).get(doc_id)
break
except ResourceNotFound:
pass
query = {"filter":
{"ids": {
"values": [doc_id]}}}
def to_json(doc):
return json.dumps(doc, indent=4, sort_keys=True) if doc else "NOT FOUND!"
found_indices = {}
doc_type = couch_doc.get('doc_type')
es_doc_type = None
for index, url in ES_URLS.items():
res = run_query(url, query)
if 'hits' in res and res['hits']['total'] == 1:
es_doc = res['hits']['hits'][0]['_source']
found_indices[index] = to_json(es_doc)
es_doc_type = es_doc_type or es_doc.get('doc_type')
doc_type = doc_type or es_doc_type or 'Unknown'
context = {
"doc_id": doc_id,
"status": "found" if found_indices else "NOT FOUND!",
"doc_type": doc_type,
"couch_doc": to_json(couch_doc),
"found_indices": found_indices,
}
return render(request, "hqadmin/doc_in_es.html", context)
def handle(self, *args, **options):
if len(args) < 2 or len(args) > 3:
raise CommandError('Usage is copy_doc %s' % self.args)
sourcedb = Database(args[0])
doc_ids_or_file = args[1]
domain = args[2] if len(args) == 3 else None
if os.path.isfile(doc_ids_or_file):
with open(doc_ids_or_file) as f:
doc_ids = f.read().splitlines()
else:
doc_ids = doc_ids_or_file.split(',')
print("Starting copy of {} docs".format(len(doc_ids)))
for doc_id in doc_ids:
print('Copying doc: {}'.format(doc_id))
doc_json = sourcedb.get(doc_id)
if domain:
doc_json['domain'] = domain
dt = DocumentTransform(doc_json, sourcedb)
save(dt, get_db())
def handle(self, *args, **options):
if len(args) < 2 or len(args) > 3:
raise CommandError('Usage is copy_doc %s' % self.args)
sourcedb = Database(args[0])
doc_ids_or_file = args[1]
domain = args[2] if len(args) == 3 else None
if os.path.isfile(doc_ids_or_file):
with open(doc_ids_or_file) as f:
doc_ids = f.read().splitlines()
else:
doc_ids = doc_ids_or_file.split(',')
print "Starting copy of {} docs".format(len(doc_ids))
for doc_id in doc_ids:
print 'Copying doc: {}'.format(doc_id)
doc_json = sourcedb.get(doc_id)
if domain:
doc_json['domain'] = domain
dt = DocumentTransform(doc_json, sourcedb)
save(dt, get_db())
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError("Usage is copy_group_data %s" % self.args)
sourcedb = Database(args[0])
group_id = args[1]
print "getting group"
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print "getting domain"
domain = Domain.wrap(
sourcedb.view("domain/domains", key=group.domain, include_docs=True, reduce=False, limit=1).one()["doc"]
)
domain.save(force_update=True)
print "getting cases"
cases = sourcedb.view(
"hqcase/by_owner",
keys=[[group.domain, group_id, False], [group.domain, group_id, True]],
wrapper=lambda row: CommCareCase.wrap(row["doc"]),
reduce=False,
include_docs=True,
).all()
self.lenient_bulk_save(CommCareCase, cases)
print "compiling xform_ids"
xform_ids = set()
for case in cases:
xform_ids.update(case.xform_ids)
print "getting xforms"
user_ids = set(group.users)
CHUNK_SIZE = 100
def form_wrapper(row):
doc = row["doc"]
doc.pop("_attachments", None)
return XFormInstance.wrap(doc)
for i, subset in enumerate(chunked(xform_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
xforms = sourcedb.all_docs(keys=list(subset), include_docs=True, wrapper=form_wrapper).all()
self.lenient_bulk_save(XFormInstance, xforms)
for xform in xforms:
user_id = xform.metadata.userID
user_ids.add(user_id)
print "getting users"
def wrap_user(row):
doc = row["doc"]
try:
return CouchUser.wrap_correctly(doc)
except Exception as e:
logging.exception("trouble with user %s" % doc["_id"])
return None
users = sourcedb.all_docs(keys=list(user_ids), include_docs=True, wrapper=wrap_user).all()
for user in users:
# if we use bulk save, django user doesn't get sync'd
user.save(force_update=True)
def handle(self, *args, **options):
raise CommandError(
'copy_group_data is currently broken. '
'Ask Danny or Ethan to fix it along the lines of '
'https://github.com/dimagi/commcare-hq/pull/9180/files#diff-9d976dc051a36a028c6604581dfbce5dR95'
)
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains', key=group.domain, include_docs=True,
reduce=False, limit=1).one()['doc']
)
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
print 'getting case ids'
with OverrideDB(CommCareCase, sourcedb):
case_ids = get_case_ids_in_domain_by_owner(
domain.name, owner_id__in=owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
for case in cases:
xform_ids.update(case.xform_ids)
self.lenient_bulk_save(CommCareCase, cases)
if not exclude_user_owned:
# also grab submissions that may not have included any case data
for user_id in group.users:
xform_ids.update(res['id'] for res in sourcedb.view(
'all_forms/view',
startkey=['submission user', domain.name, user_id],
endkey=['submission user', domain.name, user_id, {}],
reduce=False
))
print 'copying %s xforms' % len(xform_ids)
user_ids = set(group.users)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
doc.pop('external_blobs', None)
return XFormInstance.wrap(doc)
for i, subset in enumerate(chunked(xform_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
xforms = sourcedb.all_docs(
keys=list(subset),
include_docs=True,
wrapper=form_wrapper,
).all()
self.lenient_bulk_save(XFormInstance, xforms)
for xform in xforms:
user_id = xform.metadata.userID
user_ids.add(user_id)
print 'copying %s users' % len(user_ids)
def wrap_user(row):
try:
doc = row['doc']
except KeyError:
logging.exception('trouble with user result %r' % row)
return None
try:
return CouchUser.wrap_correctly(doc)
except Exception:
logging.exception('trouble with user %s' % doc['_id'])
return None
users = sourcedb.all_docs(
keys=list(user_ids),
include_docs=True,
wrapper=wrap_user,
).all()
role_ids = set([])
for user in filter(lambda u: u is not None, users):
# if we use bulk save, django user doesn't get sync'd
domain_membership = user.get_domain_membership(domain.name)
if domain_membership and domain_membership.role_id:
role_ids.add(user.domain_membership.role_id)
user.save(force_update=True)
print 'copying %s roles' % len(role_ids)
for i, subset in enumerate(chunked(role_ids, CHUNK_SIZE)):
roles = [UserRole.wrap(role['doc']) for role in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
self.lenient_bulk_save(UserRole, roles)
if options['include_sync_logs']:
print 'copying sync logs'
for user_id in user_ids:
log_ids = [res['id'] for res in sourcedb.view("phone/sync_logs_by_user",
startkey=[user_id, {}],
endkey=[user_id],
descending=True,
reduce=False,
include_docs=True
)]
print 'user: %s, logs: %s' % (user_id, len(log_ids))
for i, subset in enumerate(chunked(log_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
logs = [SyncLog.wrap(log['doc']) for log in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
self.lenient_bulk_save(SyncLog, logs)
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
exclude_user_owned = options["exclude_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains', key=group.domain, include_docs=True,
reduce=False, limit=1).one()['doc']
)
dt = DocumentTransform(domain._obj, sourcedb)
save(dt, Domain.get_db())
owners = [group_id]
if not exclude_user_owned:
owners.extend(group.users)
def keys_for_owner(domain, owner_id):
return [
[domain, owner_id, False],
[domain, owner_id, True],
]
def get_case_ids(owners):
keys = list(itertools.chain(*[keys_for_owner(domain.name, owner_id) for owner_id in owners]))
results = sourcedb.view(
'hqcase/by_owner',
keys=keys,
reduce=False,
include_docs=False,
)
return [res['id'] for res in results]
print 'getting case ids'
case_ids = get_case_ids(owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
for case in cases:
xform_ids.update(case.xform_ids)
self.lenient_bulk_save(CommCareCase, cases)
if not exclude_user_owned:
# also grab submissions that may not have included any case data
for user_id in group.users:
xform_ids.update(res['id'] for res in sourcedb.view(
'reports_forms/all_forms',
startkey=['submission user', domain.name, user_id],
endkey=['submission user', domain.name, user_id, {}],
reduce=False
))
print 'copying %s xforms' % len(xform_ids)
user_ids = set(group.users)
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
return XFormInstance.wrap(doc)
for i, subset in enumerate(chunked(xform_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
xforms = sourcedb.all_docs(
keys=list(subset),
include_docs=True,
wrapper=form_wrapper,
).all()
self.lenient_bulk_save(XFormInstance, xforms)
for xform in xforms:
user_id = xform.metadata.userID
user_ids.add(user_id)
print 'copying %s users' % len(user_ids)
def wrap_user(row):
try:
doc = row['doc']
except KeyError:
logging.exception('trouble with user result %r' % row)
return None
try:
return CouchUser.wrap_correctly(doc)
except Exception:
logging.exception('trouble with user %s' % doc['_id'])
return None
users = sourcedb.all_docs(
keys=list(user_ids),
include_docs=True,
wrapper=wrap_user,
).all()
role_ids = set([])
for user in filter(lambda u: u is not None, users):
# if we use bulk save, django user doesn't get sync'd
if user.get_domain_membership(domain.name).role_id:
role_ids.add(user.domain_membership.role_id)
user.save(force_update=True)
print 'copying %s roles' % len(role_ids)
for i, subset in enumerate(chunked(role_ids, CHUNK_SIZE)):
roles = [UserRole.wrap(role['doc']) for role in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
self.lenient_bulk_save(UserRole, roles)
if options['include_sync_logs']:
print 'copying sync logs'
for user_id in user_ids:
log_ids = [res['id'] for res in sourcedb.view("phone/sync_logs_by_user",
startkey=[user_id, {}],
endkey=[user_id],
descending=True,
reduce=False,
include_docs=True
)]
print 'user: %s, logs: %s' % (user_id, len(log_ids))
for i, subset in enumerate(chunked(log_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
logs = [SyncLog.wrap(log['doc']) for log in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)]
self.lenient_bulk_save(SyncLog, logs)
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
include_user_owned = options["include_user_owned"]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains',
key=group.domain,
include_docs=True,
reduce=False,
limit=1).one()['doc'])
domain.save(force_update=True)
owners = [group_id]
if include_user_owned:
owners.extend(group.users)
def keys_for_owner(domain, owner_id):
return [
[domain, owner_id, False],
[domain, owner_id, True],
]
def get_case_ids(owners):
keys = list(
itertools.chain(*[
keys_for_owner(domain.name, owner_id)
for owner_id in owners
]))
results = sourcedb.view(
'hqcase/by_owner',
keys=keys,
reduce=False,
include_docs=False,
)
return [res['id'] for res in results]
CHUNK_SIZE = 100
print 'getting case ids'
case_ids = get_case_ids(owners)
xform_ids = set()
print 'copying %s cases' % len(case_ids)
for i, subset in enumerate(chunked(case_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
cases = [
CommCareCase.wrap(case['doc']) for case in sourcedb.all_docs(
keys=list(subset),
include_docs=True,
)
]
for case in cases:
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError('Usage is copy_group_data %s' % self.args)
sourcedb = Database(args[0])
group_id = args[1]
print 'getting group'
group = Group.wrap(sourcedb.get(group_id))
group.save(force_update=True)
print 'getting domain'
domain = Domain.wrap(
sourcedb.view('domain/domains',
key=group.domain,
include_docs=True,
reduce=False,
limit=1).one()['doc'])
domain.save(force_update=True)
print 'getting cases'
cases = sourcedb.view(
'hqcase/by_owner',
keys=[
[group.domain, group_id, False],
[group.domain, group_id, True],
],
wrapper=lambda row: CommCareCase.wrap(row['doc']),
reduce=False,
include_docs=True).all()
self.lenient_bulk_save(CommCareCase, cases)
print 'compiling xform_ids'
xform_ids = set()
for case in cases:
xform_ids.update(case.xform_ids)
print 'getting xforms'
user_ids = set(group.users)
CHUNK_SIZE = 100
def form_wrapper(row):
doc = row['doc']
doc.pop('_attachments', None)
return XFormInstance.wrap(doc)
for i, subset in enumerate(chunked(xform_ids, CHUNK_SIZE)):
print i * CHUNK_SIZE
xforms = sourcedb.all_docs(
keys=list(subset),
include_docs=True,
wrapper=form_wrapper,
).all()
self.lenient_bulk_save(XFormInstance, xforms)
for xform in xforms:
user_id = xform.metadata.userID
user_ids.add(user_id)
print 'getting users'
def wrap_user(row):
doc = row['doc']
try:
return CouchUser.wrap_correctly(doc)
except Exception as e:
logging.exception('trouble with user %s' % doc['_id'])
return None
users = sourcedb.all_docs(keys=list(user_ids),
include_docs=True,
wrapper=wrap_user).all()
for user in users:
# if we use bulk save, django user doesn't get sync'd
user.save(force_update=True)
