class FormDocumentStore(DocumentStore):
def __init__(self, domain, xmlns=None):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
self.xmlns = xmlns
def get_document(self, doc_id):
try:
form = self.form_accessors.get_form(doc_id)
return self._to_json(form)
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
@staticmethod
def _to_json(form):
if isinstance(form, XFormInstanceSQL):
return form.to_json(include_attachments=True)
else:
return form.to_json()
def iter_document_ids(self):
return iter(self.form_accessors.iter_form_ids_by_xmlns(self.xmlns))
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
try:
yield self._to_json(wrapped_form)
except (DocumentNotFoundError, MissingFormXml):
pass
def handle(self, domains, file_name, **options):
blob_db = get_blob_db()
with open(file_name, 'w', encoding='utf-8') as csv_file:
field_names = ['domain', 'archived', 'form_id', 'received_on']
csv_writer = csv.DictWriter(csv_file, field_names)
csv_writer.writeheader()
for domain in domains:
self.stdout.write("Handling domain %s" % domain)
form_db = FormAccessors(domain)
form_ids = form_db.get_all_form_ids_in_domain()
form_ids.extend(
form_db.get_all_form_ids_in_domain('XFormArchived'))
for form in with_progress_bar(form_db.iter_forms(form_ids),
len(form_ids)):
if isinstance(form, CouchForm):
meta = form.blobs.get(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain,
form.is_archived, form.received_on,
form.form_id)
elif isinstance(form, XFormInstanceSQL):
meta = form.get_attachment_meta(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain,
form.is_archived, form.received_on,
form.form_id)
else:
raise Exception("not sure how we got here")
def handle(self, domain, folder_path, **options):
if os.path.exists(folder_path):
if not os.path.isdir(folder_path):
raise CommandError(
'Folder path must be the path to a directory')
else:
os.mkdir(folder_path)
form_accessors = FormAccessors(domain)
form_ids = form_accessors.get_all_form_ids_in_domain()
for form in form_accessors.iter_forms(form_ids):
form_path = os.path.join(folder_path, form.form_id)
if not os.path.exists(form_path):
os.mkdir(form_path)
form_meta = FormMetadata(
user_id=form.user_id,
received_on=form.received_on,
app_id=form.app_id,
build_id=form.build_id,
attachments=list(form.attachments.keys()),
auth_context=form.auth_context,
)
with open(os.path.join(form_path, 'metadata.json'), 'w') as meta:
json.dump(form_meta.to_json(), meta)
xml = form.get_xml()
with open(os.path.join(form_path, 'form.xml'), 'w') as f:
f.write(xml)
for name, meta in form.attachments.items():
with open(os.path.join(form_path, name), 'w') as f:
f.write(form.get_attachment(name))
def handle(self, domain, folder_path, **options):
if os.path.exists(folder_path):
if not os.path.isdir(folder_path):
raise CommandError('Folder path must be the path to a directory')
else:
os.mkdir(folder_path)
form_accessors = FormAccessors(domain)
form_ids = form_accessors.get_all_form_ids_in_domain()
for form in form_accessors.iter_forms(form_ids):
form_path = os.path.join(folder_path, form.form_id)
if not os.path.exists(form_path):
os.mkdir(form_path)
form_meta = FormMetadata(
user_id=form.user_id,
received_on=form.received_on,
app_id=form.app_id,
build_id=form.build_id,
attachments=form.attachments.keys(),
auth_context=form.auth_context,
)
with open(os.path.join(form_path, 'metadata.json'), 'w') as meta:
json.dump(form_meta.to_json(), meta)
xml = form.get_xml()
with open(os.path.join(form_path, 'form.xml'), 'w') as f:
f.write(xml)
for name, meta in form.attachments.items():
with open(os.path.join(form_path, name), 'w') as f:
f.write(form.get_attachment(name))
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
def get_document(self, doc_id):
try:
return add_couch_properties_to_sql_form_json(self.form_accessors.get_form(doc_id).to_json())
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def iter_document_ids(self, last_id=None):
# todo: support last_id
return iter(self.form_accessors.get_all_form_ids_in_domain())
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
yield add_couch_properties_to_sql_form_json(wrapped_form.to_json())
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
def get_document(self, doc_id):
try:
return self.form_accessors.get_form(doc_id).to_json(include_attachments=True)
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def iter_document_ids(self, last_id=None):
# todo: support last_id
return iter(self.form_accessors.get_all_form_ids_in_domain())
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
yield wrapped_form.to_json()
def handle(self, username, domain, **options):
this_form_accessor = FormAccessors(domain=domain)
user = CouchUser.get_by_username(username)
if not user:
logger.info("User {} not found.".format(username))
sys.exit(1)
user_id = user._id
form_ids = this_form_accessor.get_form_ids_for_user(user_id)
input_response = six.moves.input(
"Update {} form(s) for user {} in domain {}? (y/n): ".format(len(form_ids), username, domain))
if input_response == "y":
for form_data in this_form_accessor.iter_forms(form_ids):
form_attachment_xml_new = self.update_form_data(form_data, NEW_USERNAME)
this_form_accessor.modify_attachment_xml_and_metadata(form_data,
form_attachment_xml_new,
NEW_USERNAME)
logging.info("Updated {} form(s) for user {} in domain {}".format(len(form_ids), username, domain))
elif input_response == "n":
logging.info("No forms updated, exiting.")
else:
logging.info("Command not recognized. Exiting.")
def handle(self, username, domain, **options):
this_form_accessor = FormAccessors(domain=domain)
user = CouchUser.get_by_username(username)
if not user:
logger.info("User {} not found.".format(username))
sys.exit(1)
user_id = user._id
form_ids = this_form_accessor.get_form_ids_for_user(user_id)
input_response = six.moves.input(
"Update {} form(s) for user {} in domain {}? (y/n): ".format(
len(form_ids), username, domain))
if input_response == "y":
for form_data in this_form_accessor.iter_forms(form_ids):
form_attachment_xml_new = self.update_form_data(
form_data, NEW_USERNAME)
this_form_accessor.modify_attachment_xml_and_metadata(
form_data, form_attachment_xml_new, NEW_USERNAME)
logging.info("Updated {} form(s) for user {} in domain {}".format(
len(form_ids), username, domain))
elif input_response == "n":
logging.info("No forms updated, exiting.")
else:
logging.info("Command not recognized. Exiting.")
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain, xmlns=None):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
self.xmlns = xmlns
def get_document(self, doc_id):
try:
form = self.form_accessors.get_form(doc_id)
if isinstance(form, XFormInstanceSQL):
return form.to_json(include_attachments=True)
else:
return form.to_json()
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def iter_document_ids(self, last_id=None):
# todo: support last_id
return iter(self.form_accessors.iter_form_ids_by_xmlns(self.xmlns))
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
yield wrapped_form.to_json()
def handle(self, domains, file_name, **options):
blob_db = get_blob_db()
with open(file_name, 'w', encoding='utf-8') as csv_file:
field_names = ['domain', 'archived', 'form_id', 'received_on']
csv_writer = csv.DictWriter(csv_file, field_names)
csv_writer.writeheader()
for domain in domains:
self.stdout.write("Handling domain %s" % domain)
form_db = FormAccessors(domain)
form_ids = form_db.get_all_form_ids_in_domain()
form_ids.extend(form_db.get_all_form_ids_in_domain('XFormArchived'))
for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)):
if isinstance(form, CouchForm):
meta = form.blobs.get(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id)
elif isinstance(form, XFormInstanceSQL):
meta = form.get_attachment_meta(ATTACHMENT_NAME)
if not meta or not blob_db.exists(key=meta.key):
self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id)
else:
raise Exception("not sure how we got here")
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore):
def __init__(self, domain, xmlns=None):
self.domain = domain
self.form_accessors = FormAccessors(domain=domain)
self.xmlns = xmlns
def get_document(self, doc_id):
try:
form = self.form_accessors.get_form(doc_id)
if isinstance(form, XFormInstanceSQL):
return form.to_json(include_attachments=True)
else:
return form.to_json()
except (XFormNotFound, BlobError) as e:
raise DocumentNotFoundError(e)
def iter_document_ids(self, last_id=None):
# todo: support last_id
return iter(self.form_accessors.iter_form_ids_by_xmlns(self.xmlns))
def iter_documents(self, ids):
for wrapped_form in self.form_accessors.iter_forms(ids):
yield wrapped_form.to_json()
class Command(BaseCommand):
help = """Expects 4 arguments in order : domain app_id version_number test_run
ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes
domain is included with app_id to ensure the user knows what app to delete
version_number to delete data accumulated by versions BEFORE this : integer
test_run should be yes(case-sensitive) for a test_run and anything otherwise
though deletion would be re-confirmed so dont panic
"""
def add_arguments(self, parser):
parser.add_argument('domain')
parser.add_argument('app_id')
parser.add_argument('version_number', type=int)
parser.add_argument('test_run')
def __init__(self):
super(Command, self).__init__()
self.case_ids = set()
self.filtered_xform_ids, self.xform_ids = [], []
self.xform_writer, self.case_writer = None, None
self.forms_accessor, self.case_accessors = None, None
self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None
self.version_mapping = dict()
def setup(self):
self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b'))
self.xform_writer.writerow(XFORM_HEADER)
self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b'))
self.case_writer.writerow(CASE_HEADER)
self.forms_accessor = FormAccessors(self.domain)
self.case_accessors = CaseAccessors(self.domain)
def ensure_prerequisites(self, domain, app_id, version_number, test_run):
self.domain = domain
self.app_id = app_id
self.version_number = version_number
self.test_run = test_run == 'yes'
_notify_parsed_args(domain, app_id, version_number, test_run)
app = Application.get(self.app_id)
if app.domain != self.domain:
raise CommandError('Domain not same as from app id')
self.setup()
def handle(self, domain, app_id, version_number, test_run, **options):
self.ensure_prerequisites(domain, app_id, version_number, test_run)
self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain()
self.iterate_forms_and_collect_case_ids()
_print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids)
if self.data_to_delete() and self.delete_permitted():
self.delete_forms_and_cases()
print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME))
else:
print('Process Finished w/o Changes..')
def iterate_forms_and_collect_case_ids(self):
print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids))
for xform in self.forms_accessor.iter_forms(self.xform_ids):
# Get app version by fetching app corresponding to xform build_id since xform.form
# does not have updated app version unless form was updated for that version
app_version_built_with = self.get_xform_build_version(xform)
if app_version_built_with and app_version_built_with < self.version_number:
_print_form_details(xform, self.xform_writer, app_version_built_with)
self.ensure_valid_xform(xform)
self.filtered_xform_ids.append(xform.form_id)
self.case_ids = self.case_ids.union(get_case_ids_from_form(xform))
else:
print('skipping xform id: %s' % xform.form_id)
if self.case_ids:
self.print_case_details()
def get_xform_build_version(self, xform):
version_from_mapping = None
if xform.build_id:
version_from_mapping = self.version_mapping.get(xform.build_id, None)
if not version_from_mapping:
try:
get_app_version = get_app(self.domain, xform.build_id).version
except Http404:
get_app_version = None
if get_app_version:
version_from_mapping = int(get_app_version)
self.version_mapping[xform.build_id] = version_from_mapping
return version_from_mapping
def ensure_valid_xform(self, xform):
if xform.app_id != self.app_id and xform.domain != self.domain:
_raise_xform_domain_mismatch(xform)
def print_case_details(self):
for case in self.case_accessors.iter_cases(self.case_ids):
_print_case_details(case, self.case_writer)
def delete_permitted(self):
return not self.test_run and are_you_sure()
def data_to_delete(self):
return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0
def delete_forms_and_cases(self):
print('Proceeding with deleting forms and cases')
self.forms_accessor.soft_delete_forms(self.filtered_xform_ids)
self.case_accessors.soft_delete_cases(list(self.case_ids))
class Command(BaseCommand):
help = """Expects 4 arguments in order : domain app_id version_number test_run
ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes
domain is included with app_id to ensure the user knows what app to delete
version_number to delete data accumulated by versions BEFORE this : integer
test_run should be yes(case-sensitive) for a test_run and anything otherwise
though deletion would be re-confirmed so dont panic
"""
def add_arguments(self, parser):
parser.add_argument('domain')
parser.add_argument('app_id')
parser.add_argument('version_number', type=int)
parser.add_argument('test_run')
def __init__(self):
super(Command, self).__init__()
self.case_ids = set()
self.filtered_xform_ids, self.xform_ids = [], []
self.xform_writer, self.case_writer = None, None
self.forms_accessor, self.case_accessors = None, None
self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None
self.version_mapping = dict()
def setup(self):
self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b'))
self.xform_writer.writerow(XFORM_HEADER)
self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b'))
self.case_writer.writerow(CASE_HEADER)
self.forms_accessor = FormAccessors(self.domain)
self.case_accessors = CaseAccessors(self.domain)
def ensure_prerequisites(self, domain, app_id, version_number, test_run):
NotAllowed.check(domain)
self.domain = domain
self.app_id = app_id
self.version_number = version_number
self.test_run = test_run == 'yes'
_notify_parsed_args(domain, app_id, version_number, test_run)
app = Application.get(self.app_id)
if app.domain != self.domain:
raise CommandError('Domain not same as from app id')
self.setup()
def handle(self, domain, app_id, version_number, test_run, **options):
self.ensure_prerequisites(domain, app_id, version_number, test_run)
self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain()
self.iterate_forms_and_collect_case_ids()
_print_final_debug_info(self.xform_ids, self.filtered_xform_ids,
self.case_ids)
if self.data_to_delete() and self.delete_permitted():
self.delete_forms_and_cases()
print("Process Completed!! Keep copy of files %s, %s" %
(XFORM_FILENAME, CASE_FILE_NAME))
else:
print('Process Finished w/o Changes..')
def iterate_forms_and_collect_case_ids(self):
print("Iterating Through %s XForms and Collecting Case Ids" %
len(self.xform_ids))
for xform in self.forms_accessor.iter_forms(self.xform_ids):
# Get app version by fetching app corresponding to xform build_id since xform.form
# does not have updated app version unless form was updated for that version
app_version_built_with = self.get_xform_build_version(xform)
if app_version_built_with and app_version_built_with < self.version_number:
_print_form_details(xform, self.xform_writer,
app_version_built_with)
self.ensure_valid_xform(xform)
self.filtered_xform_ids.append(xform.form_id)
self.case_ids = self.case_ids.union(
get_case_ids_from_form(xform))
else:
print('skipping xform id: %s' % xform.form_id)
if self.case_ids:
self.print_case_details()
def get_xform_build_version(self, xform):
version_from_mapping = None
if xform.build_id:
version_from_mapping = self.version_mapping.get(
xform.build_id, None)
if not version_from_mapping:
try:
get_app_version = get_app(self.domain,
xform.build_id).version
except Http404:
get_app_version = None
if get_app_version:
version_from_mapping = int(get_app_version)
self.version_mapping[xform.build_id] = version_from_mapping
return version_from_mapping
def ensure_valid_xform(self, xform):
if xform.app_id != self.app_id and xform.domain != self.domain:
_raise_xform_domain_mismatch(xform)
def print_case_details(self):
for case in self.case_accessors.iter_cases(self.case_ids):
_print_case_details(case, self.case_writer)
def delete_permitted(self):
return not self.test_run and are_you_sure()
def data_to_delete(self):
return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0
def delete_forms_and_cases(self):
print('Proceeding with deleting forms and cases')
self.forms_accessor.soft_delete_forms(self.filtered_xform_ids)
self.case_accessors.soft_delete_cases(list(self.case_ids))
