def change_form_unique_id(form, map):
unique_id = form['unique_id']
new_unique_id = map.get(form['xmlns'], random_hex())
form['unique_id'] = new_unique_id
if ("%s.xml" % unique_id) in app_source['_attachments']:
app_source['_attachments']["%s.xml" % new_unique_id] = app_source['_attachments'].pop("%s.xml" % unique_id)
return new_unique_id
def delete_all_data(request, domain, template="cleanup/delete_all_data.html"):
if request.method == 'GET':
return render(request, template, {
'domain': domain
})
key = make_form_couch_key(domain)
xforms = XFormInstance.view('reports_forms/all_forms',
startkey=key,
endkey=key+[{}],
include_docs=True,
reduce=False
)
cases = CommCareCase.view('case/by_date_modified',
startkey=[domain, {}, {}],
endkey=[domain, {}, {}, {}],
include_docs=True,
reduce=False
)
suffix = DELETED_SUFFIX
deletion_id = random_hex()
for thing_list in (xforms, cases):
for thing in thing_list:
thing.doc_type += suffix
thing['-deletion_id'] = deletion_id
thing.save()
return HttpResponseRedirect(reverse('homepage'))
def test_aggregate_ucr_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
aggregate_table_definition = AggregateTableDefinition.objects.create(
domain=domain_name,
primary_data_source_id=uuid.uuid4(),
table_id=random_hex(),
)
secondary_table_definition = SecondaryTableDefinition.objects.create(
table_definition=aggregate_table_definition,
data_source_id=uuid.uuid4(),
)
PrimaryColumn.objects.create(table_definition=aggregate_table_definition)
SecondaryColumn.objects.create(table_definition=secondary_table_definition)
self._assert_aggregate_ucr_count(domain_name, 1)
self.domain.delete()
self._assert_aggregate_ucr_count(self.domain.name, 0)
self._assert_aggregate_ucr_count(self.domain2.name, 1)
self.assertEqual(SecondaryTableDefinition.objects.count(), 1)
self.assertEqual(
SecondaryTableDefinition.objects.filter(table_definition__domain=self.domain2.name).count(),
1
)
self.assertEqual(PrimaryColumn.objects.count(), 1)
self.assertEqual(PrimaryColumn.objects.filter(table_definition__domain=self.domain2.name).count(), 1)
self.assertEqual(SecondaryColumn.objects.count(), 1)
self.assertEqual(
SecondaryColumn.objects.filter(table_definition__table_definition__domain=self.domain2.name).count(),
1
)
def update_unique_ids(app_source):
from corehq.apps.app_manager.models import form_id_references, jsonpath_update
app_source = deepcopy(app_source)
def change_form_unique_id(form):
unique_id = form['unique_id']
new_unique_id = random_hex()
form['unique_id'] = new_unique_id
if ("%s.xml" % unique_id) in app_source['_attachments']:
app_source['_attachments']["%s.xml" % new_unique_id] = app_source['_attachments'].pop("%s.xml" % unique_id)
return new_unique_id
change_form_unique_id(app_source['user_registration'])
id_changes = {}
for m, module in enumerate(app_source['modules']):
for f, form in enumerate(module['forms']):
old_id = form['unique_id']
new_id = change_form_unique_id(app_source['modules'][m]['forms'][f])
id_changes[old_id] = new_id
for reference_path in form_id_references:
for reference in reference_path.find(app_source):
if reference.value in id_changes:
jsonpath_update(reference, id_changes[reference.value])
for module in app_source['modules']:
if module['module_type'] == 'report':
for report_config in module['report_configs']:
report_config['uuid'] = random_hex()
return app_source
def change_form_unique_id(form):
unique_id = form['unique_id']
new_unique_id = random_hex()
form['unique_id'] = new_unique_id
if ("%s.xml" % unique_id) in app_source['_attachments']:
app_source['_attachments']["%s.xml" % new_unique_id] = app_source['_attachments'].pop("%s.xml" % unique_id)
return new_unique_id
def test_aggregate_ucr_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
aggregate_table_definition = AggregateTableDefinition.objects.create(
domain=domain_name,
primary_data_source_id=uuid.uuid4(),
table_id=random_hex(),
)
secondary_table_definition = SecondaryTableDefinition.objects.create(
table_definition=aggregate_table_definition,
data_source_id=uuid.uuid4(),
)
PrimaryColumn.objects.create(
table_definition=aggregate_table_definition)
SecondaryColumn.objects.create(
table_definition=secondary_table_definition)
self._assert_aggregate_ucr_count(domain_name, 1)
self.domain.delete()
self._assert_aggregate_ucr_count(self.domain.name, 0)
self._assert_aggregate_ucr_count(self.domain2.name, 1)
self.assertEqual(SecondaryTableDefinition.objects.count(), 1)
self.assertEqual(
SecondaryTableDefinition.objects.filter(
table_definition__domain=self.domain2.name).count(), 1)
self.assertEqual(PrimaryColumn.objects.count(), 1)
self.assertEqual(
PrimaryColumn.objects.filter(
table_definition__domain=self.domain2.name).count(), 1)
self.assertEqual(SecondaryColumn.objects.count(), 1)
self.assertEqual(
SecondaryColumn.objects.filter(
table_definition__table_definition__domain=self.domain2.name).
count(), 1)
def from_log_record(cls, record):
type = exc = tb = traceback_string = None
if record.exc_info:
type, exc, tb = record.exc_info
traceback_string = "".join(traceback.format_tb(tb))
# ['args', 'created', 'exc_info', 'exc_text', 'filename', 'funcName',
# 'getMessage', 'levelname', 'levelno', 'lineno', 'module', 'msecs',
# 'msg', 'name', 'pathname', 'process', 'processName',
# 'relativeCreated', 'thread', 'threadName']
# to fix a problem with gunicorn
# in which things get logged after fork
# but before calling random.seed(), we
# call it here
random.seed()
c_record = ExceptionRecord(
function=record.funcName,
line_number=record.lineno,
level=record.levelname,
logger_name=record.name,
pathname=record.pathname,
message=record.getMessage(),
type=str(type),
stack_trace=traceback_string,
date=datetime.utcnow(),
url="",
query_params={},
)
# couchdbkit's uuid generation is not fork-safe
# so we generate a random id
c_record._id = random_hex()
c_record.save()
# fire signal
signals.couchlog_created.send(sender="couchlog", record=c_record)
return c_record
def test_data_interfaces(self):
for domain_name in [self.domain.name, self.domain2.name]:
automatic_update_rule = AutomaticUpdateRule.objects.create(domain=domain_name)
AutomaticUpdateAction.objects.create(rule=automatic_update_rule)
AutomaticUpdateRuleCriteria.objects.create(rule=automatic_update_rule)
CaseRuleAction.objects.create(rule=automatic_update_rule)
CaseRuleCriteria.objects.create(rule=automatic_update_rule)
CaseRuleSubmission.objects.create(
created_on=datetime.utcnow(),
domain=domain_name,
form_id=random_hex(),
rule=automatic_update_rule,
)
DomainCaseRuleRun.objects.create(domain=domain_name, started_on=datetime.utcnow())
self._assert_data_interfaces(domain_name, 1)
self.domain.delete()
self._assert_data_interfaces(self.domain.name, 0)
self._assert_data_interfaces(self.domain2.name, 1)
self.assertEqual(AutomaticUpdateAction.objects.count(), 1)
self.assertEqual(AutomaticUpdateAction.objects.filter(rule__domain=self.domain2.name).count(), 1)
self.assertEqual(AutomaticUpdateRuleCriteria.objects.count(), 1)
self.assertEqual(AutomaticUpdateRuleCriteria.objects.filter(rule__domain=self.domain2.name).count(), 1)
self.assertEqual(CaseRuleAction.objects.count(), 1)
self.assertEqual(CaseRuleAction.objects.filter(rule__domain=self.domain2.name).count(), 1)
self.assertEqual(CaseRuleCriteria.objects.count(), 1)
self.assertEqual(CaseRuleCriteria.objects.filter(rule__domain=self.domain2.name).count(), 1)
def _create_form(app_id, xmlns, with_attachment):
xml = FORM_XML.format(doc_id=random_hex(), xmlns=xmlns)
attachments = {}
if with_attachment:
attachment = open('./corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg', 'rb')
attachments = {
'pic.jpg': UploadedFile(attachment, 'pic.jpg')
}
submit_form_locally(xml, domain=DOMAIN, app_id=app_id, attachments=attachments)
def _create_form(app_id, xmlns, with_attachment):
xml = FORM_XML.format(doc_id=random_hex(), xmlns=xmlns)
attachments = {}
if with_attachment:
attachment = open(
'./corehq/ex-submodules/casexml/apps/case/tests/data/attachments/fruity.jpg',
'rb')
attachments = {'pic.jpg': UploadedFile(attachment, 'pic.jpg')}
submit_form_locally(xml,
domain=DOMAIN,
app_id=app_id,
attachments=attachments)
def test_userreports_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
AsyncIndicator.objects.create(
domain=domain_name,
doc_id=random_hex(),
doc_type='doc_type',
indicator_config_ids=[],
)
self._assert_userreports_counts(domain_name, 1)
self.domain.delete()
self._assert_userreports_counts(self.domain.name, 0)
self._assert_userreports_counts(self.domain2.name, 1)
def test_userreports_delete(self):
for domain_name in [self.domain.name, self.domain2.name]:
AsyncIndicator.objects.create(
domain=domain_name,
doc_id=random_hex(),
doc_type='doc_type',
indicator_config_ids=[],
)
self._assert_userreports_counts(domain_name, 1)
self.domain.delete()
self._assert_userreports_counts(self.domain.name, 0)
self._assert_userreports_counts(self.domain2.name, 1)
def check_elasticsearch():
cluster_health = check_es_cluster_health()
if cluster_health == 'red':
return ServiceStatus(False, "Cluster health at %s" % cluster_health)
doc = {'_id': 'elasticsearch-service-check-{}'.format(random_hex()[:7]),
'date': datetime.datetime.now().isoformat()}
try:
send_to_elasticsearch('groups', doc)
refresh_elasticsearch_index('groups')
hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits
if doc in hits:
return ServiceStatus(True, "Successfully sent a doc to ES and read it back")
else:
return ServiceStatus(False, "Something went wrong sending a doc to ES")
finally:
send_to_elasticsearch('groups', doc, delete=True) # clean up
def check_elasticsearch():
cluster_health = check_es_cluster_health()
if cluster_health != 'green':
return ServiceStatus(False, "Cluster health at %s" % cluster_health)
doc = {'_id': 'elasticsearch-service-check-{}'.format(random_hex()[:7]),
'date': datetime.datetime.now().isoformat()}
try:
send_to_elasticsearch('groups', doc)
refresh_elasticsearch_index('groups')
hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits
if doc in hits:
return ServiceStatus(True, "Successfully sent a doc to ES and read it back")
else:
return ServiceStatus(False, "Something went wrong sending a doc to ES")
finally:
send_to_elasticsearch('groups', doc, delete=True) # clean up
def test_case_importer(self):
for domain_name in [self.domain.name, self.domain2.name]:
case_upload_record = CaseUploadRecord.objects.create(
domain=domain_name,
task_id=uuid.uuid4().hex,
upload_id=uuid.uuid4().hex,
)
CaseUploadFormRecord.objects.create(
case_upload_record=case_upload_record,
form_id=random_hex(),
)
self._assert_case_importer_counts(domain_name, 1)
self.domain.delete()
self._assert_case_importer_counts(self.domain.name, 0)
self._assert_case_importer_counts(self.domain2.name, 1)
self.assertEqual(CaseUploadFormRecord.objects.count(), 1)
self.assertEqual(
CaseUploadFormRecord.objects.filter(
case_upload_record__domain=self.domain2.name).count(), 1)
def retire(self):
suffix = DELETED_SUFFIX
deletion_id = random_hex()
# doc_type remains the same, since the views use base_doc instead
if not self.base_doc.endswith(suffix):
self.base_doc += suffix
self['-deletion_id'] = deletion_id
for form in self.get_forms():
form.doc_type += suffix
form['-deletion_id'] = deletion_id
form.save()
for case in self.get_cases():
case.doc_type += suffix
case['-deletion_id'] = deletion_id
case.save()
try:
django_user = self.get_django_user()
except User.DoesNotExist:
pass
else:
django_user.delete()
self.save()
def test_data_interfaces(self):
for domain_name in [self.domain.name, self.domain2.name]:
automatic_update_rule = AutomaticUpdateRule.objects.create(domain=domain_name)
CaseRuleAction.objects.create(rule=automatic_update_rule)
CaseRuleCriteria.objects.create(rule=automatic_update_rule)
CaseRuleSubmission.objects.create(
created_on=datetime.utcnow(),
domain=domain_name,
form_id=random_hex(),
rule=automatic_update_rule,
)
DomainCaseRuleRun.objects.create(domain=domain_name, started_on=datetime.utcnow())
self._assert_data_interfaces(domain_name, 1)
self.domain.delete()
self._assert_data_interfaces(self.domain.name, 0)
self._assert_data_interfaces(self.domain2.name, 1)
self.assertEqual(CaseRuleAction.objects.count(), 1)
self.assertEqual(CaseRuleAction.objects.filter(rule__domain=self.domain2.name).count(), 1)
self.assertEqual(CaseRuleCriteria.objects.count(), 1)
self.assertEqual(CaseRuleCriteria.objects.filter(rule__domain=self.domain2.name).count(), 1)
def test_case_importer(self):
for domain_name in [self.domain.name, self.domain2.name]:
case_upload_record = CaseUploadRecord.objects.create(
domain=domain_name,
task_id=uuid.uuid4().hex,
upload_id=uuid.uuid4().hex,
)
CaseUploadFormRecord.objects.create(
case_upload_record=case_upload_record,
form_id=random_hex(),
)
self._assert_case_importer_counts(domain_name, 1)
self.domain.delete()
self._assert_case_importer_counts(self.domain.name, 0)
self._assert_case_importer_counts(self.domain2.name, 1)
self.assertEqual(CaseUploadFormRecord.objects.count(), 1)
self.assertEqual(
CaseUploadFormRecord.objects.filter(case_upload_record__domain=self.domain2.name).count(),
1
)
def toggle_js_user_cachebuster(username):
return random_hex()[:3]
def toggle_js_domain_cachebuster(domain):
# to get fresh cachebusters on the next deploy
# change the date below (output from *nix `date` command)
# Mon Oct 31 10:30:09 EDT 2016
return random_hex()[:3]
def toggle_js_domain_cachebuster(domain):
# to get fresh cachebusters on the next deploy
# change the date below (output from *nix `date` command)
# Wed Apr 25 14:12:12 EDT 2018
return random_hex()[:3]
def toggle_js_domain_cachebuster(domain):
# to get fresh cachebusters on the next deploy
# change the date below (output from *nix `date` command)
# Thu Mar 3 16:21:30 EST 2016
return random_hex()[:3]
def toggle_js_user_cachebuster(username):
return random_hex()[:3]
def toggle_js_domain_cachebuster(domain):
# to get fresh cachebusters on the next deploy
# change the date below (output from *nix `date` command)
# Mon Oct 31 10:30:09 EDT 2016
return random_hex()[:3]
def toggle_js_domain_cachebuster(domain):
# to get fresh cachebusters on the next deploy
# change the date below (output from *nix `date` command)
# Thu Mar 3 16:21:30 EST 2016
return random_hex()[:3]