class FormDocumentStore(DocumentStore): def __init__(self, domain, xmlns=None): self.domain = domain self.form_accessors = FormAccessors(domain=domain) self.xmlns = xmlns def get_document(self, doc_id): try: form = self.form_accessors.get_form(doc_id) return self._to_json(form) except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e) @staticmethod def _to_json(form): if isinstance(form, XFormInstanceSQL): return form.to_json(include_attachments=True) else: return form.to_json() def iter_document_ids(self): return iter(self.form_accessors.iter_form_ids_by_xmlns(self.xmlns)) def iter_documents(self, ids): for wrapped_form in self.form_accessors.iter_forms(ids): try: yield self._to_json(wrapped_form) except (DocumentNotFoundError, MissingFormXml): pass
def handle(self, domains, file_name, **options): blob_db = get_blob_db() with open(file_name, 'w', encoding='utf-8') as csv_file: field_names = ['domain', 'archived', 'form_id', 'received_on'] csv_writer = csv.DictWriter(csv_file, field_names) csv_writer.writeheader() for domain in domains: self.stdout.write("Handling domain %s" % domain) form_db = FormAccessors(domain) form_ids = form_db.get_all_form_ids_in_domain() form_ids.extend( form_db.get_all_form_ids_in_domain('XFormArchived')) for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)): if isinstance(form, CouchForm): meta = form.blobs.get(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) elif isinstance(form, XFormInstanceSQL): meta = form.get_attachment_meta(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) else: raise Exception("not sure how we got here")
def handle(self, domain, folder_path, **options): if os.path.exists(folder_path): if not os.path.isdir(folder_path): raise CommandError( 'Folder path must be the path to a directory') else: os.mkdir(folder_path) form_accessors = FormAccessors(domain) form_ids = form_accessors.get_all_form_ids_in_domain() for form in form_accessors.iter_forms(form_ids): form_path = os.path.join(folder_path, form.form_id) if not os.path.exists(form_path): os.mkdir(form_path) form_meta = FormMetadata( user_id=form.user_id, received_on=form.received_on, app_id=form.app_id, build_id=form.build_id, attachments=list(form.attachments.keys()), auth_context=form.auth_context, ) with open(os.path.join(form_path, 'metadata.json'), 'w') as meta: json.dump(form_meta.to_json(), meta) xml = form.get_xml() with open(os.path.join(form_path, 'form.xml'), 'w') as f: f.write(xml) for name, meta in form.attachments.items(): with open(os.path.join(form_path, name), 'w') as f: f.write(form.get_attachment(name))
def handle(self, domain, folder_path, **options): if os.path.exists(folder_path): if not os.path.isdir(folder_path): raise CommandError('Folder path must be the path to a directory') else: os.mkdir(folder_path) form_accessors = FormAccessors(domain) form_ids = form_accessors.get_all_form_ids_in_domain() for form in form_accessors.iter_forms(form_ids): form_path = os.path.join(folder_path, form.form_id) if not os.path.exists(form_path): os.mkdir(form_path) form_meta = FormMetadata( user_id=form.user_id, received_on=form.received_on, app_id=form.app_id, build_id=form.build_id, attachments=form.attachments.keys(), auth_context=form.auth_context, ) with open(os.path.join(form_path, 'metadata.json'), 'w') as meta: json.dump(form_meta.to_json(), meta) xml = form.get_xml() with open(os.path.join(form_path, 'form.xml'), 'w') as f: f.write(xml) for name, meta in form.attachments.items(): with open(os.path.join(form_path, name), 'w') as f: f.write(form.get_attachment(name))
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain): self.domain = domain self.form_accessors = FormAccessors(domain=domain) def get_document(self, doc_id): try: return add_couch_properties_to_sql_form_json(self.form_accessors.get_form(doc_id).to_json()) except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e) def iter_document_ids(self, last_id=None): # todo: support last_id return iter(self.form_accessors.get_all_form_ids_in_domain()) def iter_documents(self, ids): for wrapped_form in self.form_accessors.iter_forms(ids): yield add_couch_properties_to_sql_form_json(wrapped_form.to_json())
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain): self.domain = domain self.form_accessors = FormAccessors(domain=domain) def get_document(self, doc_id): try: return self.form_accessors.get_form(doc_id).to_json(include_attachments=True) except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e) def iter_document_ids(self, last_id=None): # todo: support last_id return iter(self.form_accessors.get_all_form_ids_in_domain()) def iter_documents(self, ids): for wrapped_form in self.form_accessors.iter_forms(ids): yield wrapped_form.to_json()
def handle(self, username, domain, **options): this_form_accessor = FormAccessors(domain=domain) user = CouchUser.get_by_username(username) if not user: logger.info("User {} not found.".format(username)) sys.exit(1) user_id = user._id form_ids = this_form_accessor.get_form_ids_for_user(user_id) input_response = six.moves.input( "Update {} form(s) for user {} in domain {}? (y/n): ".format(len(form_ids), username, domain)) if input_response == "y": for form_data in this_form_accessor.iter_forms(form_ids): form_attachment_xml_new = self.update_form_data(form_data, NEW_USERNAME) this_form_accessor.modify_attachment_xml_and_metadata(form_data, form_attachment_xml_new, NEW_USERNAME) logging.info("Updated {} form(s) for user {} in domain {}".format(len(form_ids), username, domain)) elif input_response == "n": logging.info("No forms updated, exiting.") else: logging.info("Command not recognized. Exiting.")
def handle(self, username, domain, **options): this_form_accessor = FormAccessors(domain=domain) user = CouchUser.get_by_username(username) if not user: logger.info("User {} not found.".format(username)) sys.exit(1) user_id = user._id form_ids = this_form_accessor.get_form_ids_for_user(user_id) input_response = six.moves.input( "Update {} form(s) for user {} in domain {}? (y/n): ".format( len(form_ids), username, domain)) if input_response == "y": for form_data in this_form_accessor.iter_forms(form_ids): form_attachment_xml_new = self.update_form_data( form_data, NEW_USERNAME) this_form_accessor.modify_attachment_xml_and_metadata( form_data, form_attachment_xml_new, NEW_USERNAME) logging.info("Updated {} form(s) for user {} in domain {}".format( len(form_ids), username, domain)) elif input_response == "n": logging.info("No forms updated, exiting.") else: logging.info("Command not recognized. Exiting.")
class ReadonlyFormDocumentStore(ReadOnlyDocumentStore): def __init__(self, domain, xmlns=None): self.domain = domain self.form_accessors = FormAccessors(domain=domain) self.xmlns = xmlns def get_document(self, doc_id): try: form = self.form_accessors.get_form(doc_id) if isinstance(form, XFormInstanceSQL): return form.to_json(include_attachments=True) else: return form.to_json() except (XFormNotFound, BlobError) as e: raise DocumentNotFoundError(e) def iter_document_ids(self, last_id=None): # todo: support last_id return iter(self.form_accessors.iter_form_ids_by_xmlns(self.xmlns)) def iter_documents(self, ids): for wrapped_form in self.form_accessors.iter_forms(ids): yield wrapped_form.to_json()
def handle(self, domains, file_name, **options): blob_db = get_blob_db() with open(file_name, 'w', encoding='utf-8') as csv_file: field_names = ['domain', 'archived', 'form_id', 'received_on'] csv_writer = csv.DictWriter(csv_file, field_names) csv_writer.writeheader() for domain in domains: self.stdout.write("Handling domain %s" % domain) form_db = FormAccessors(domain) form_ids = form_db.get_all_form_ids_in_domain() form_ids.extend(form_db.get_all_form_ids_in_domain('XFormArchived')) for form in with_progress_bar(form_db.iter_forms(form_ids), len(form_ids)): if isinstance(form, CouchForm): meta = form.blobs.get(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) elif isinstance(form, XFormInstanceSQL): meta = form.get_attachment_meta(ATTACHMENT_NAME) if not meta or not blob_db.exists(key=meta.key): self.write_row(csv_writer, domain, form.is_archived, form.received_on, form.form_id) else: raise Exception("not sure how we got here")
class Command(BaseCommand): help = """Expects 4 arguments in order : domain app_id version_number test_run ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes domain is included with app_id to ensure the user knows what app to delete version_number to delete data accumulated by versions BEFORE this : integer test_run should be yes(case-sensitive) for a test_run and anything otherwise though deletion would be re-confirmed so dont panic """ def add_arguments(self, parser): parser.add_argument('domain') parser.add_argument('app_id') parser.add_argument('version_number', type=int) parser.add_argument('test_run') def __init__(self): super(Command, self).__init__() self.case_ids = set() self.filtered_xform_ids, self.xform_ids = [], [] self.xform_writer, self.case_writer = None, None self.forms_accessor, self.case_accessors = None, None self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None self.version_mapping = dict() def setup(self): self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b')) self.xform_writer.writerow(XFORM_HEADER) self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b')) self.case_writer.writerow(CASE_HEADER) self.forms_accessor = FormAccessors(self.domain) self.case_accessors = CaseAccessors(self.domain) def ensure_prerequisites(self, domain, app_id, version_number, test_run): self.domain = domain self.app_id = app_id self.version_number = version_number self.test_run = test_run == 'yes' _notify_parsed_args(domain, app_id, version_number, test_run) app = Application.get(self.app_id) if app.domain != self.domain: raise CommandError('Domain not same as from app id') self.setup() def handle(self, domain, app_id, version_number, test_run, **options): self.ensure_prerequisites(domain, app_id, version_number, test_run) self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain() self.iterate_forms_and_collect_case_ids() _print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids) if self.data_to_delete() and self.delete_permitted(): self.delete_forms_and_cases() print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME)) else: print('Process Finished w/o Changes..') def iterate_forms_and_collect_case_ids(self): print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids)) for xform in self.forms_accessor.iter_forms(self.xform_ids): # Get app version by fetching app corresponding to xform build_id since xform.form # does not have updated app version unless form was updated for that version app_version_built_with = self.get_xform_build_version(xform) if app_version_built_with and app_version_built_with < self.version_number: _print_form_details(xform, self.xform_writer, app_version_built_with) self.ensure_valid_xform(xform) self.filtered_xform_ids.append(xform.form_id) self.case_ids = self.case_ids.union(get_case_ids_from_form(xform)) else: print('skipping xform id: %s' % xform.form_id) if self.case_ids: self.print_case_details() def get_xform_build_version(self, xform): version_from_mapping = None if xform.build_id: version_from_mapping = self.version_mapping.get(xform.build_id, None) if not version_from_mapping: try: get_app_version = get_app(self.domain, xform.build_id).version except Http404: get_app_version = None if get_app_version: version_from_mapping = int(get_app_version) self.version_mapping[xform.build_id] = version_from_mapping return version_from_mapping def ensure_valid_xform(self, xform): if xform.app_id != self.app_id and xform.domain != self.domain: _raise_xform_domain_mismatch(xform) def print_case_details(self): for case in self.case_accessors.iter_cases(self.case_ids): _print_case_details(case, self.case_writer) def delete_permitted(self): return not self.test_run and are_you_sure() def data_to_delete(self): return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0 def delete_forms_and_cases(self): print('Proceeding with deleting forms and cases') self.forms_accessor.soft_delete_forms(self.filtered_xform_ids) self.case_accessors.soft_delete_cases(list(self.case_ids))
class Command(BaseCommand): help = """Expects 4 arguments in order : domain app_id version_number test_run ex: ./manage.py purge_forms_and_cases testapp c531daeece0633738c9a3676a13e3d4f 88 yes domain is included with app_id to ensure the user knows what app to delete version_number to delete data accumulated by versions BEFORE this : integer test_run should be yes(case-sensitive) for a test_run and anything otherwise though deletion would be re-confirmed so dont panic """ def add_arguments(self, parser): parser.add_argument('domain') parser.add_argument('app_id') parser.add_argument('version_number', type=int) parser.add_argument('test_run') def __init__(self): super(Command, self).__init__() self.case_ids = set() self.filtered_xform_ids, self.xform_ids = [], [] self.xform_writer, self.case_writer = None, None self.forms_accessor, self.case_accessors = None, None self.domain, self.app_id, self.version_number, self.test_run = None, None, None, None self.version_mapping = dict() def setup(self): self.xform_writer = csv.writer(open(XFORM_FILENAME, 'w+b')) self.xform_writer.writerow(XFORM_HEADER) self.case_writer = csv.writer(open(CASE_FILE_NAME, 'w+b')) self.case_writer.writerow(CASE_HEADER) self.forms_accessor = FormAccessors(self.domain) self.case_accessors = CaseAccessors(self.domain) def ensure_prerequisites(self, domain, app_id, version_number, test_run): NotAllowed.check(domain) self.domain = domain self.app_id = app_id self.version_number = version_number self.test_run = test_run == 'yes' _notify_parsed_args(domain, app_id, version_number, test_run) app = Application.get(self.app_id) if app.domain != self.domain: raise CommandError('Domain not same as from app id') self.setup() def handle(self, domain, app_id, version_number, test_run, **options): self.ensure_prerequisites(domain, app_id, version_number, test_run) self.xform_ids = self.forms_accessor.get_all_form_ids_in_domain() self.iterate_forms_and_collect_case_ids() _print_final_debug_info(self.xform_ids, self.filtered_xform_ids, self.case_ids) if self.data_to_delete() and self.delete_permitted(): self.delete_forms_and_cases() print("Process Completed!! Keep copy of files %s, %s" % (XFORM_FILENAME, CASE_FILE_NAME)) else: print('Process Finished w/o Changes..') def iterate_forms_and_collect_case_ids(self): print("Iterating Through %s XForms and Collecting Case Ids" % len(self.xform_ids)) for xform in self.forms_accessor.iter_forms(self.xform_ids): # Get app version by fetching app corresponding to xform build_id since xform.form # does not have updated app version unless form was updated for that version app_version_built_with = self.get_xform_build_version(xform) if app_version_built_with and app_version_built_with < self.version_number: _print_form_details(xform, self.xform_writer, app_version_built_with) self.ensure_valid_xform(xform) self.filtered_xform_ids.append(xform.form_id) self.case_ids = self.case_ids.union( get_case_ids_from_form(xform)) else: print('skipping xform id: %s' % xform.form_id) if self.case_ids: self.print_case_details() def get_xform_build_version(self, xform): version_from_mapping = None if xform.build_id: version_from_mapping = self.version_mapping.get( xform.build_id, None) if not version_from_mapping: try: get_app_version = get_app(self.domain, xform.build_id).version except Http404: get_app_version = None if get_app_version: version_from_mapping = int(get_app_version) self.version_mapping[xform.build_id] = version_from_mapping return version_from_mapping def ensure_valid_xform(self, xform): if xform.app_id != self.app_id and xform.domain != self.domain: _raise_xform_domain_mismatch(xform) def print_case_details(self): for case in self.case_accessors.iter_cases(self.case_ids): _print_case_details(case, self.case_writer) def delete_permitted(self): return not self.test_run and are_you_sure() def data_to_delete(self): return len(self.filtered_xform_ids) != 0 or len(self.case_ids) != 0 def delete_forms_and_cases(self): print('Proceeding with deleting forms and cases') self.forms_accessor.soft_delete_forms(self.filtered_xform_ids) self.case_accessors.soft_delete_cases(list(self.case_ids))