def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = sorted(form.form_id for form in forms)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms(
'other_domain', [other_form.form_id]))
attachments = sorted(get_blob_db().metadb.get_for_parents(form_ids),
key=lambda meta: meta.parent_id)
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(
DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(BlobNotFound):
attachment.open()
with attachments[0].open() as content:
self.assertIsNotNone(content.read())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = sorted(form.form_id for form in forms)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = sorted(
get_blob_db().metadb.get_for_parents(form_ids),
key=lambda meta: meta.parent_id
)
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(BlobNotFound):
attachment.open()
with attachments[0].open() as content:
self.assertIsNotNone(content.read())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True))
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(AttachmentNotFound):
attachment.read_content()
self.assertIsNotNone(attachments[0].read_content())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def handle(self, domain, **options):
NotAllowed.check(domain)
if not options['noinput']:
confirm = input(
"""
Are you sure you want to hard delete all forms and cases in domain "{}"?
This operation is PERMANENT.
Type the domain's name again to continue, or anything else to cancel:
""".format(domain)
)
if confirm != domain:
print("\n\t\tDomain deletion cancelled.")
return
logger.info('Hard deleting forms...')
deleted_sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain)
for form_id_chunk in chunked(with_progress_bar(deleted_sql_form_ids, stream=silence_during_tests()), 500):
FormAccessorSQL.hard_delete_forms(domain, list(form_id_chunk), delete_attachments=True)
logger.info('Hard deleting cases...')
deleted_sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)
for case_id_chunk in chunked(with_progress_bar(deleted_sql_case_ids, stream=silence_during_tests()), 500):
CaseAccessorSQL.hard_delete_cases(domain, list(case_id_chunk))
logger.info('Done.')
def test_hard_delete_forms_and_attachments(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
attachments = list(FormAccessorSQL.get_attachments_for_forms(form_ids, ordered=True))
self.assertEqual(3, len(attachments))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
for attachment in attachments[1:]:
with self.assertRaises(AttachmentNotFound):
attachment.read_content()
self.assertIsNotNone(attachments[0].read_content())
other_form = FormAccessorSQL.get_form(other_form.form_id)
self.assertIsNotNone(other_form.get_xml())
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
for form in xforms:
form.state |= XFormInstanceSQL.DELETED
publish_form_saved(form)
case.deleted = True
publish_case_saved(case)
def form_context(form_ids):
for form_id in form_ids:
builder = FormSubmissionBuilder(
form_id=form_id,
metadata=TestFormMetadata(domain=DOMAIN),
)
submit_form_locally(builder.as_xml_string(), DOMAIN)
try:
yield
finally:
FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids)
def test_hard_delete_forms(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
def test_hard_delete_forms(self):
forms = [create_form_for_test(DOMAIN) for i in range(3)]
form_ids = [form.form_id for form in forms]
other_form = create_form_for_test('other_domain')
self.addCleanup(lambda: FormAccessorSQL.hard_delete_forms('other_domain', [other_form.form_id]))
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(3, len(forms))
deleted = FormAccessorSQL.hard_delete_forms(DOMAIN, form_ids[1:] + [other_form.form_id])
self.assertEqual(2, deleted)
forms = FormAccessorSQL.get_forms(form_ids)
self.assertEqual(1, len(forms))
self.assertEqual(form_ids[0], forms[0].form_id)
def blow_away_migration(domain, state_dir):
assert not should_use_sql_backend(domain)
delete_state_db(domain, state_dir)
log.info("deleting forms...")
for form_ids in iter_chunks(XFormInstanceSQL, "form_id", domain):
FormAccessorSQL.hard_delete_forms(domain, form_ids, delete_attachments=False)
log.info("deleting cases...")
for case_ids in iter_chunks(CommCareCaseSQL, "case_id", domain):
CaseAccessorSQL.hard_delete_cases(domain, case_ids)
log.info("blew away migration for domain %s\n", domain)
def _blow_away_migration(domain):
assert not should_use_sql_backend(domain)
delete_diff_db(domain)
for doc_type in doc_types():
sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
def _blow_away_migration(domain):
assert not should_use_sql_backend(domain)
delete_diff_db(domain)
for doc_type in doc_types():
sql_form_ids = FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_form_ids = FormAccessorSQL.get_deleted_form_ids_in_domain(domain)
FormAccessorSQL.hard_delete_forms(domain, sql_form_ids, delete_attachments=False)
sql_case_ids = CaseAccessorSQL.get_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
sql_case_ids = CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)
CaseAccessorSQL.hard_delete_cases(domain, sql_case_ids)
class FormProcessorSQL(object):
@classmethod
def store_attachments(cls, xform, attachments):
xform_attachments = []
for attachment in attachments:
xform_attachment = XFormAttachmentSQL(
name=attachment.name,
attachment_id=uuid.uuid4(),
content_type=attachment.content_type,
)
xform_attachment.write_content(attachment.content)
if xform_attachment.is_image:
try:
img_size = Image.open(attachment.content_as_file()).size
xform_attachment.properties = dict(width=img_size[0],
height=img_size[1])
except IOError:
xform_attachment.content_type = 'application/octet-stream'
xform_attachments.append(xform_attachment)
xform.unsaved_attachments = xform_attachments
@classmethod
def new_xform(cls, form_data):
form_id = extract_meta_instance_id(form_data) or unicode(uuid.uuid4())
return XFormInstanceSQL(
# other properties can be set post-wrap
form_id=form_id,
xmlns=form_data.get('@xmlns'),
received_on=datetime.datetime.utcnow(),
user_id=extract_meta_user_id(form_data),
)
@classmethod
def is_duplicate(cls, xform_id, domain=None):
return FormAccessorSQL.form_exists(xform_id, domain=domain)
@classmethod
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
for form in xforms:
form.state |= XFormInstanceSQL.DELETED
publish_form_saved(form)
case.deleted = True
publish_case_saved(case)
class FormProcessorSQL(object):
@classmethod
def store_attachments(cls, xform, attachments):
xform.attachments_list = attachments
@classmethod
def copy_attachments(cls, from_form, to_form):
to_form.copy_attachments(from_form)
@classmethod
def copy_form_operations(cls, from_form, to_form):
for op in from_form.history:
op.id = None
op.form = to_form
to_form.track_create(op)
@classmethod
def new_xform(cls, form_data):
form_id = extract_meta_instance_id(form_data) or str(uuid.uuid4())
return XFormInstanceSQL(
# other properties can be set post-wrap
form_id=form_id,
xmlns=form_data.get('@xmlns'),
received_on=datetime.datetime.utcnow(),
user_id=extract_meta_user_id(form_data),
)
@classmethod
def is_duplicate(cls, xform_id, domain=None):
return FormAccessorSQL.form_exists(xform_id, domain=domain)
@classmethod
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
for form in xforms:
form.state |= XFormInstanceSQL.DELETED
publish_form_saved(form)
case.deleted = True
publish_case_saved(case)
def tearDownClass(cls):
FormAccessorSQL.hard_delete_forms(SQL_DOMAIN, [cls.form_id])
super().tearDownClass()
def hard_delete_case_and_forms(cls, domain, case, xforms):
form_ids = [xform.form_id for xform in xforms]
FormAccessorSQL.hard_delete_forms(domain, form_ids)
CaseAccessorSQL.hard_delete_cases(domain, [case.case_id])
forms = []
for i in range(3):
case_id = uuid.uuid4().hex
case_block = factory.get_case_block(case_id, case_type='case_type')
form, [case] = factory.post_case_blocks([case_block])
cases.append(case)
forms.append(form)
case_ids = [case.case_id for case in cases]
form_ids = [form.form_id for form in forms]
try:
yield form_ids, case_ids
finally:
if should_use_sql_backend('domain'):
FormAccessorSQL.hard_delete_forms('domain', form_ids)
CaseAccessorSQL.hard_delete_cases('domain', case_ids)
else:
for case in cases:
case.delete()
for form in forms:
form.delete()
@contextmanager
def test_domain():
from corehq.apps.domain.shortcuts import create_domain
domain = create_domain('domain')
try:
yield domain
finally:
