def eval_dots_block(xform_json, callback=None): """ Evaluate the dots block in the xform submission and put it in the computed_ block for the xform. """ case_id = get_case_id(xform_json) do_continue = False #first, set the pact_data to json if the dots update stuff is there. try: if xform_json.get(PACT_DOTS_DATA_PROPERTY, {}).has_key('processed'): #already processed, skipping return xform_json[PACT_DOTS_DATA_PROPERTY] = {} if not isinstance(xform_json['form']['case'].get('update', None), dict): #no case update property, skipping pass else: #update is a dict if xform_json['form']['case']['update'].has_key('dots'): dots_json = xform_json['form']['case']['update']['dots'] if isinstance(dots_json, str) or isinstance(dots_json, unicode): json_data = simplejson.loads(dots_json) xform_json[PACT_DOTS_DATA_PROPERTY]['dots'] = json_data do_continue=True else: #no dots data in doc pass xform_json[PACT_DOTS_DATA_PROPERTY]['processed']=datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') XFormInstance.get_db().save_doc(xform_json) except Exception, ex: #if this gets triggered, that's ok because web entry don't got them tb = traceback.format_exc() notify_exception(None, message="PACT error evaluating DOTS block docid %s, %s\n\tTraceback: %s" % (xform_json['_id'], ex, tb))
def save_processed_models(cls, processed_forms, cases=None, stock_updates=None): docs = list(processed_forms) + (cases or []) docs = filter(None, docs) assert XFormInstance.get_db().uri == CommCareCase.get_db().uri XFormInstance.get_db().bulk_save(docs) for stock_update in stock_updates or []: stock_update.commit()
def test_assign_noop(self): self._make_tree() num_forms = XFormInstance.get_db().view('hqadmin/forms_over_time').all()[0]['value'] res = assign_case(self.primary, self.original_owner._id, include_subcases=True, include_parent_cases=True) self.assertEqual(0, len(res)) new_num_forms = XFormInstance.get_db().view('hqadmin/forms_over_time').all()[0]['value'] self.assertEqual(new_num_forms, num_forms)
def tag_forms_as_deleted_rebuild_associated_cases(user_id, domain, form_id_list, deletion_id, deletion_date, deleted_cases=None): """ Upon user deletion, mark associated forms as deleted and prep cases for a rebuild. - 2 saves/sec for cloudant slowness (rate_limit) """ if deleted_cases is None: deleted_cases = set() cases_to_rebuild = set() forms_to_check = get_docs(XFormInstance.get_db(), form_id_list) forms_to_save = [] for form in forms_to_check: assert form['domain'] == domain if not is_deleted(form): form['doc_type'] += DELETED_SUFFIX form['-deletion_id'] = deletion_id form['-deletion_date'] = deletion_date forms_to_save.append(form) # rebuild all cases anyways since we don't know if this has run or not if the task was killed cases_to_rebuild.update(get_case_ids_from_form(form)) XFormInstance.get_db().bulk_save(forms_to_save) detail = UserArchivedRebuild(user_id=user_id) for case in cases_to_rebuild - deleted_cases: _rebuild_case_with_retries.delay(domain, case, detail)
def update_analytics_indexes(): """ Mostly for testing; wait until analytics data sources are up to date so that calls to analytics functions return up-to-date """ XFormInstance.get_db().view("reports_forms/all_forms", limit=1).all() XFormInstance.get_db().view("exports_forms/by_xmlns", limit=1).all()
def test_basic_edit(self): xml_data1, xml_data2 = self._get_files() docs = [] doc = post_xform_to_couch(xml_data1) self.assertEqual(self.ID, doc.get_id) self.assertEqual("XFormInstance", doc.doc_type) self.assertEqual("", doc.form["vitals"]["height"]) self.assertEqual("other", doc.form["assessment"]["categories"]) doc.domain = "test-domain" doc.save() doc = post_xform_to_couch(xml_data2, domain="test-domain") self.assertEqual(self.ID, doc.get_id) self.assertEqual("XFormInstance", doc.doc_type) self.assertEqual("100", doc.form["vitals"]["height"]) self.assertEqual("Edited Baby!", doc.form["assessment"]["categories"]) docs.append(doc) doc = XFormDeprecated.view("couchforms/edits", include_docs=True).first() self.assertEqual(self.ID, doc.orig_id) self.assertNotEqual(self.ID, doc.get_id) self.assertEqual(XFormDeprecated.__name__, doc.doc_type) self.assertEqual("", doc.form["vitals"]["height"]) self.assertEqual("other", doc.form["assessment"]["categories"]) self.assertEqual(XFormInstance.get_db().fetch_attachment(doc.get_id, "form.xml"), xml_data1) self.assertEqual(XFormInstance.get_db().fetch_attachment(self.ID, "form.xml"), xml_data2) for doc in docs: doc.delete()
def process_cases(xform, config=None): """ Creates or updates case objects which live outside of the form. If reconcile is true it will perform an additional step of reconciling the case update history after the case is processed. """ warnings.warn( 'This function is deprecated. You should be using SubmissionPost.', DeprecationWarning, ) assert getattr(settings, 'UNIT_TESTING', False) domain = get_and_check_xform_domain(xform) with CaseDbCache(domain=domain, lock=True, deleted_ok=True) as case_db: case_result = process_cases_with_casedb([xform], case_db, config=config) cases = case_result.cases docs = [xform] + cases now = datetime.datetime.utcnow() for case in cases: case.server_modified_on = now XFormInstance.get_db().bulk_save(docs) for case in cases: case_post_save.send(CommCareCase, case=case) case_result.commit_dirtiness_flags() return cases
def save_processed_models(cls, processed_forms, cases=None, stock_result=None): docs = list(processed_forms) + (cases or []) docs = filter(None, docs) assert XFormInstance.get_db().uri == CommCareCase.get_db().uri with bulk_atomic_blobs(docs): XFormInstance.get_db().bulk_save(docs) if stock_result: stock_result.commit()
def resave_form(domain, form): from corehq.form_processor.utils import should_use_sql_backend from corehq.form_processor.change_publishers import publish_form_saved from couchforms.models import XFormInstance if should_use_sql_backend(domain): publish_form_saved(form) else: XFormInstance.get_db().save_doc(form.to_json())
def tearDown(self): try: XFormInstance.get_db().delete_doc(self.ID) except ResourceNotFound: pass deprecated_xforms = XFormDeprecated.view("couchforms/edits", include_docs=True).all() for form in deprecated_xforms: form.delete()
def tearDown(self): try: XFormInstance.get_db().delete_doc(self.ID) except ResourceNotFound: pass deprecated_xforms = access_edits(include_docs=True).all() for form in deprecated_xforms: form.delete()
def is_duplicate(cls, xform_id, domain=None): if domain: try: existing_doc = XFormInstance.get_db().get(xform_id) except ResourceNotFound: return False return existing_doc.get('domain') == domain and existing_doc.get('doc_type') in doc_types() else: return xform_id in XFormInstance.get_db()
def test_wrong_domain(self): domain = 'test-domain' XFormInstance.get_db().save_doc({ '_id': self.ID, 'doc_type': 'XFormInstance', 'domain': 'wrong-domain', }) doc = post_xform_to_couch(instance=self._get_file(), domain=domain) self.assertNotEqual(doc.get_id, self.ID)
def test_broken_save(self): """ Test that if the second form submission terminates unexpectedly and the main form isn't saved, then there are no side effects such as the original having been marked as deprecated. """ class BorkDB(object): """context manager for making a db's bulk_save temporarily fail""" def __init__(self, db): self.old = {} self.db = db def __enter__(self): self.old['bulk_save'] = self.db.bulk_save self.db.bulk_save = MagicMock(name='bulk_save', side_effect=RequestFailed()) def __exit__(self, exc_type, exc_val, exc_tb): self.db.bulk_save = self.old['bulk_save'] self.assertEqual( XFormInstance.view('couchforms/edits', key=self.ID).count(), 0) self.assertFalse(XFormInstance.get_db().doc_exist(self.ID)) xml_data1, xml_data2 = self._get_files() submit_form_locally(xml_data1, self.domain) doc = XFormInstance.get(self.ID) self.assertEqual(self.ID, doc.get_id) self.assertEqual("XFormInstance", doc.doc_type) self.assertEqual(self.domain, doc.domain) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(), 0 ) with BorkDB(XFormInstance.get_db()): with self.assertRaises(RequestFailed): submit_form_locally(xml_data2, self.domain) # it didn't go through, so make sure there are no edits still self.assertEqual( XFormInstance.view('couchforms/edits', key=self.ID).count(), 0) self.assertTrue(XFormInstance.get_db().doc_exist(self.ID)) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID, saved=False).count(), 1 ) self.assertEqual( UnfinishedSubmissionStub.objects.filter(xform_id=self.ID).count(), 1 )
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format(domain, status)) db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set(FormAccessorSQL.get_form_ids_in_domain_by_type(domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) form_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "XFormInstance-Deleted", XFormInstance.get_db()) ) form_ids_in_sql = set(FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status( "XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase", (0, 0)) has_diffs |= self._print_status( 'CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) case_ids_in_couch = set(get_doc_ids_in_domain_by_type( domain, "CommCareCase-Deleted", XFormInstance.get_db()) ) case_ids_in_sql = set(CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop("CommCareCase-Deleted", (0, 0)) has_diffs |= self._print_status( 'CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only ) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status( key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only ) if diffs_only and not has_diffs: print(shell_green("No differences found between old and new docs!")) return has_diffs
def test_wrong_doc_type(self): domain = 'test-domain' id = '7H46J37FGH3' XFormInstance.get_db().save_doc({ '_id': id, 'doc_type': 'Foo', 'domain': domain, }) doc = post_xform_to_couch(instance=self._get_file(), domain=domain) self.assertNotEqual(doc.get_id, id)
def test_wrong_doc_type(self): domain = 'test-domain' XFormInstance.get_db().save_doc({ '_id': self.ID, 'doc_type': 'Foo', 'domain': domain, }) self.addCleanup(lambda: XFormInstance.get_db().delete_doc(self.ID)) instance = self._get_file() xform = FormProcessorInterface.post_xform(instance, domain=domain) self.assertNotEqual(xform.id, self.ID)
def handle(self, *args, **options): ids = get_form_ids_by_type('ipm-senegal', 'XFormInstance') to_save = [] locations = SQLLocation.objects.filter(domain='ipm-senegal').values_list('location_id', 'name') locations_map = {location_id: name for (location_id, name) in locations} for doc in iter_docs(XFormInstance.get_db(), ids): try: if 'PPS_name' in doc['form'] and not doc['form']['PPS_name']: case = SupplyPointCase.get(doc['form']['case']['@case_id']) if case.type == 'supply-point': print 'Updating XFormInstance:', doc['_id'] pps_name = locations_map[case.location_id] instance = XFormInstance.get(doc['_id']) # fix the XFormInstance instance.form['PPS_name'] = pps_name for instance_prod in instance.form['products']: instance_prod['PPS_name'] = instance_prod['PPS_name'] or pps_name # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) pps_name_node = xml_object.find(re.sub('}.*', '}PPS_name', xml_object.tag)) pps_name_node.text = pps_name products_nodes = xml_object.findall(re.sub('}.*', '}products', xml_object.tag)) for product_node in products_nodes: product_pps_name_node = product_node.find(re.sub('}.*', '}PPS_name', xml_object.tag)) product_pps_name_node.text = pps_name updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder(instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml']['content_type'] ) instance._attachments = attachment_builder.to_json() to_save.append(instance) except Exception: print 'Failed to save XFormInstance:', doc['_id'] if len(to_save) > 500: XFormInstance.get_db().bulk_save(to_save) to_save = [] if to_save: XFormInstance.get_db().bulk_save(to_save)
def test_simple_delete(self): factory = CaseFactory() case = factory.create_case() [case] = factory.create_or_update_case(CaseStructure(case_id=case._id, attrs={'update': {'foo': 'bar'}})) self.assertTrue(CommCareCase.get_db().doc_exist(case._id)) self.assertEqual(2, len(case.xform_ids)) for form_id in case.xform_ids: self.assertTrue(XFormInstance.get_db().doc_exist(form_id)) safe_hard_delete(case) self.assertFalse(CommCareCase.get_db().doc_exist(case._id)) for form_id in case.xform_ids: self.assertFalse(XFormInstance.get_db().doc_exist(form_id))
def save_processed_models(cls, processed_forms, cases=None, stock_result=None): docs = list(processed_forms) for form in docs: if form: form.server_modified_on = datetime.datetime.utcnow() docs += (cases or []) docs = [_f for _f in docs if _f] assert XFormInstance.get_db().uri == CommCareCase.get_db().uri with bulk_atomic_blobs(docs): XFormInstance.get_db().bulk_save(docs) if stock_result: stock_result.commit()
def tag_forms_as_deleted_rebuild_associated_cases(formlist, deletion_id, deleted_cases=None): from casexml.apps.case.cleanup import rebuild_case if deleted_cases is None: deleted_cases = set() cases_to_rebuild = set() for form in formlist: form['doc_type'] += DELETED_SUFFIX form['-deletion_id'] = deletion_id cases_to_rebuild.update(get_case_ids_from_form(form)) XFormInstance.get_db().bulk_save(formlist) for case in cases_to_rebuild - deleted_cases: rebuild_case(case)
def form_multimedia_export(request, domain, app_id): try: xmlns = request.GET["xmlns"] startdate = request.GET["startdate"] enddate = request.GET["enddate"] zip_name = request.GET.get("name", None) except KeyError: return HttpResponseBadRequest() def filename(form, question_id, extension): return "%s-%s-%s-%s.%s" % (form['form']['@name'], unidecode(question_id), form['form']['meta']['username'], form['_id'], extension) key = [domain, app_id, xmlns] stream_file = cStringIO.StringIO() zf = zipfile.ZipFile(stream_file, mode='w', compression=zipfile.ZIP_STORED) size = 22 # overhead for a zipfile unknown_number = 0 form_ids = {f['id'] for f in XFormInstance.get_db().view("attachments/attachments", start_key=key + [startdate], end_key=key + [enddate, {}], reduce=False)} for form in iter_docs(XFormInstance.get_db(), form_ids): f = XFormInstance.wrap(form) if not zip_name: zip_name = unidecode(form['form']['@name']) for key in form['_attachments'].keys(): if form['_attachments'][key]['content_type'] == 'text/xml': continue extension = unicode(os.path.splitext(key)[1]) try: question_id = unicode('-'.join(find_question_id(form['form'], key))) except TypeError: question_id = unicode('unknown' + str(unknown_number)) unknown_number += 1 fname = filename(form, question_id, extension) zi = zipfile.ZipInfo(fname, parse(form['received_on']).timetuple()) zf.writestr(zi, f.fetch_attachment(key, stream=True).read()) # includes overhead for file in zipfile size += f['_attachments'][key]['length'] + 88 + 2 * len(fname) zf.close() response = HttpResponse(stream_file.getvalue(), mimetype="application/zip") response['Content-Length'] = size response['Content-Disposition'] = 'attachment; filename=%s.zip' % zip_name return response
def historical_forms(request, domain): assert request.couch_user.is_member_of(domain) user_id = request.couch_user.get_id db = XFormInstance.get_db() form_ids = { f['id'] for f in db.view( 'reports_forms/all_forms', startkey=["submission user", domain, user_id], endkey=["submission user", domain, user_id, {}], reduce=False, ) } def data(): yield ( '<OpenRosaResponse xmlns="http://openrosa.org/http/response" ' 'items="{}">\n <message nature="success"/>\n' .format(len(form_ids)) ) for form_id in form_ids: # this is a hack to call this method # Should only hit couch once per form, to get the attachment xml = XFormInstance(_id=form_id).get_xml_element() if xml: yield ' {}'.format(etree.tostring(xml)) else: yield ' <XFormNotFound/>' yield '\n' yield '</OpenRosaResponse>\n' # to make this not stream, just call list on data() return HttpResponse(data(), content_type='application/xml')
def handle(self, *args, **options): if len(args) < 2: print "please specify a filepath and an archiving_user" return filepath = args[0] archiving_user = args[1] try: form_ids = open(filepath) except Exception as e: print "there was an issue opening the file: %s" % e return try: form_ids = [f[0] for f in csv.reader(form_ids)] except Exception as e: print "there was an issue reading the file %s" % e return for xform_doc in iter_docs(XFormInstance.get_db(), form_ids): try: xform = XFormInstance.wrap(xform_doc) xform.archive(user_id=archiving_user) print "Archived form %s in domain %s" % ( xform._id, xform.domain ) except Exception as e: print "Issue archiving XFORM %s for domain %s: %s" % ( xform_doc['_id'], xform_doc['domain'], e )
def get_value(self, user_ids, datespan=None, is_debug=False): if datespan: enddate = datespan.enddate_utc else: enddate = datetime.datetime.utcnow() days = [] for user_id in user_ids: key = make_form_couch_key(self.domain, user_id=user_id) results = XFormInstance.get_db().view("reports_forms/all_forms", reduce=False, include_docs=False, descending=True, startkey=key+[enddate.isoformat(), {}], endkey=key, limit=1 ).first() try: last_transmission = results['key'][-1] last_date = dateutil.parser.parse(last_transmission) last_date = last_date.replace(tzinfo=pytz.utc) enddate = enddate.replace(tzinfo=pytz.utc) td = enddate - last_date days.append(td.days) except Exception: pass if len(days) == 1: return days[0] if not days: return None return days
def testUpdateWithNoNewAttachment(self): _, case = self._doCreateCaseWithMultimedia() bulk_save = XFormInstance.get_db().bulk_save bulk_save_attachments = [] # pull out and record attachments to docs being bulk saved def new_bulk_save(docs, *args, **kwargs): for doc in docs: if doc['_id'] == TEST_CASE_ID: bulk_save_attachments.append(doc['_attachments']) bulk_save(docs, *args, **kwargs) self._doSubmitUpdateWithMultimedia( new_attachments=[], removes=[]) with patch('couchforms.models.XFormInstance._db.bulk_save', new_bulk_save): # submit from the 2 min in the past to trigger a rebuild self._doSubmitUpdateWithMultimedia( new_attachments=[], removes=[], date=datetime.utcnow() - timedelta(minutes=2)) # make sure there's exactly one bulk save recorded self.assertEqual(len(bulk_save_attachments), 1) # make sure none of the attachments were re-saved in rebuild self.assertEqual( [key for key, value in bulk_save_attachments[0].items() if value.get('data')], [])
def archive_forms(domain, user, uploaded_data): response = {"errors": [], "success": []} form_ids = [row.get("form_id") for row in uploaded_data] missing_forms = set(form_ids) for xform_doc in iter_docs(XFormInstance.get_db(), form_ids): xform = XFormInstance.wrap(xform_doc) missing_forms.discard(xform["_id"]) if xform["domain"] != domain: response["errors"].append( _(u"XFORM {form_id} does not belong to domain {domain}").format( form_id=xform["_id"], domain=xform["domain"] ) ) continue xform_string = _(u"XFORM {form_id} for domain {domain} by user '{username}'").format( form_id=xform["_id"], domain=xform["domain"], username=user.username ) try: xform.archive(user=user.username) response["success"].append(_(u"Successfully archived {form}").format(form=xform_string)) except Exception as e: response["errors"].append(_(u"Could not archive {form}: {error}").format(form=xform_string, error=e)) for missing_form_id in missing_forms: response["errors"].append(_(u"Could not find XForm {form_id}").format(form_id=missing_form_id)) return response
def rows(self): domain = self.report.filter_values['domain'] startdate = self.report.filter_values['startdate'] enddate = self.report.filter_values['enddate'] key_base = 'submission xmlns user' # todo this will do one couch view hit per relevant user. could be optimized to sql or something if desired user_ids = self.report.get_user_ids() rows = [] for user in user_ids: last_submission = XFormInstance.get_db().view('all_forms/view', startkey=[key_base, domain, self.xmlns, user, enddate], endkey=[key_base, domain, self.xmlns, user, startdate], limit=1, reduce=False, include_docs=True, descending=True, ).one() if last_submission: wrapped = XFormInstance.wrap(last_submission['doc']) user_row = [wrapped.get_data(path) for path in self.column_slugs] else: user_row = [NO_VALUE] * len(self.column_slugs) rows.append((user, user_row)) # format formatted_rows = list(self.report.format.format_output(rows)) # transpose return [[_(col)] + [r[i] for r in formatted_rows] for i, col in enumerate(self.column_slugs)]
def delete_all_xforms(cls, domain=None, user_id=None): view = 'couchforms/all_submissions_by_domain' view_kwargs = {} if domain and user_id: view = 'reports_forms/all_forms' view_kwargs = { 'startkey': ['submission user', domain, user_id], 'endkey': ['submission user', domain, user_id, {}], } elif domain: view_kwargs = { 'startkey': [domain], 'endkey': [domain, {}] } cls._delete_all( XFormInstance.get_db(), view, **view_kwargs ) query = XFormInstanceSQL.objects if domain is not None: query = query.filter(domain=domain) if user_id is not None: query = query.filter(user_id=user_id) query.all().delete()
def test_case_pillow_indicators(self): since = get_current_seq(XFormInstance.get_db()) self._save_doc_to_db('indicator_form.json', XFormInstance) case_id = self._save_doc_to_db('indicator_case.json', CommCareCase) case_instance = CommCareCase.get(case_id) # FormDataInCaseIndicatorDef (For those forgotten properties) forgotten_property = FormDataInCaseIndicatorDefinition.increment_or_create_unique( INDICATOR_TEST_NAMESPACE, INDICATOR_TEST_DOMAIN, slug='club_name', question_id='location.club', case_type='song_tag', xmlns='http://openrosa.org/formdesigner/indicator-create-xmlns', ) forgotten_property.save() self.case_pillow.process_changes(since=since, forever=False) indicator_case = IndicatorCase.get(case_id) self.assertEqual(indicator_case.get_id, case_instance.get_id) self.assertNotEqual( indicator_case.get_db().dbname, case_instance.get_db().dbname ) self.assertNotEqual(indicator_case.computed_, {})
def get_main_cache_invalidation_pillow(pillow_id, **kwargs): from couchforms.models import XFormInstance return _get_cache_invalidation_pillow(pillow_id, XFormInstance.get_db(), couch_filter="hqadmin/not_case_form")
def _clear_docs(self): config = ExportConfiguration( XFormInstance.get_db(), [DOMAIN, "http://www.commcarehq.org/export/test"]) for form in config.get_docs(): XFormInstance.wrap(form).delete()
def tearDownClass(cls): CommCareCase.get_db().bulk_delete(cls.cases) XFormInstance.get_db().bulk_delete(cls.forms) super(DBAccessorsTest, cls).tearDownClass()
def print_stats(self, domain, short=True, diffs_only=False): status = get_couch_sql_migration_status(domain) print("Couch to SQL migration status for {}: {}".format( domain, status)) db = open_state_db(domain, self.state_dir) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) if doc_type == "XFormInstance": form_ids_in_couch.update( get_doc_ids_in_domain_by_type(domain, "HQSubmission", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status(doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) form_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop( "XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) ZERO = Counts(0, 0) if db.has_doc_counts(): doc_counts = db.get_doc_counts() couch_missing_cases = doc_counts.get("CommCareCase-couch", ZERO).missing else: doc_counts = None couch_missing_cases = 0 for doc_type in CASE_DOC_TYPES: if doc_counts is not None: counts = doc_counts.get(doc_type, ZERO) case_ids_in_couch = db.get_missing_doc_ids( doc_type) if counts.missing else set() case_ids_in_sql = counts elif doc_type == "CommCareCase": case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set( CaseAccessorSQL.get_case_ids_in_domain(domain)) elif doc_type == "CommCareCase-Deleted": case_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db())) case_ids_in_sql = set( CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) else: raise NotImplementedError(doc_type) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status( doc_type, case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only, ) if doc_type == "CommCareCase" and couch_missing_cases: has_diffs = True print( shell_red("%s cases could not be loaded from Couch" % couch_missing_cases)) if not short: for case_id in db.get_missing_doc_ids( "CommCareCase-couch"): print(case_id) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status(key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only) if diffs_only and not has_diffs: print( shell_green("No differences found between old and new docs!")) return has_diffs
def get(self, *args, **kwargs): """ Download prior progress note submissions for local access """ db = XFormInstance.get_db() couch_user = CouchUser.from_django_user(self.request.user) username = couch_user.raw_username if hasattr(localsettings, 'debug_pact_user'): username = getattr(localsettings, 'debug_pact_user')(username) offset = 0 limit_count = 200 total_count = 0 query = { "query": { "filtered": { "filter": { "and": [{ "term": { "domain.exact": "pact" } }, { "term": { "form.#type": "progress_note" } }, { "term": { "form.meta.username": username } }] }, "query": { "match_all": {} } } }, "sort": { "received_on": "asc" }, "size": limit_count, "fields": ['_id', 'external_blobs'] } query['script_fields'] = {} query['script_fields'].update(pact_script_fields()) query['script_fields'].update(case_script_field()) res = self.xform_es.run_query(query) my_patients_ever_submitted_query = query_per_case_submissions_facet( PACT_DOMAIN, username) patients_res = self.xform_es.run_query( my_patients_ever_submitted_query) #filter by active/discharged? #get all the forms #get all the patients #get all patients to determine which to filter. active_patients = [] for pt in []: #if pt.hp_status == "Discharged": #continue case_id = pt['script_case_id'] active_patients.append(case_id) def return_iterator(): yield "<restoredata>" for result in res['hits']['hits']: data_row = result['fields'] # if data_row['script_case_id'] not in active_patients: # continue try: xml_str = (BlobHelper( data_row, db, CODES.form_xml).fetch_attachment('form.xml').replace( "<?xml version=\'1.0\' ?>", '').replace( "<?xml version='1.0' encoding='UTF-8' ?>", '')) yield xml_str except Exception as ex: logging.error( "for downloader: error fetching attachment: %s" % ex) yield "</restoredata>" response = HttpResponse(return_iterator(), content_type='text/xml') return response
def delete_all_xforms(): _delete_all(XFormInstance.get_db(), 'couchforms/all_submissions_by_domain')
assert getattr(settings, 'UNIT_TESTING', False) domain = get_and_check_xform_domain(xform) with CaseDbCacheCouch(domain=domain, lock=True, deleted_ok=True) as case_db: case_result = process_cases_with_casedb([xform], case_db, config=config) cases = case_result.cases docs = [xform] + cases now = datetime.datetime.utcnow() for case in cases: case.server_modified_on = now XFormInstance.get_db().bulk_save(docs) for case in cases: case_post_save.send(CommCareCase, case=case) case_result.commit_dirtiness_flags() return cases def reprocess_form_cases(form): """ For a given form, reprocess all case elements inside it. This operation should be a no-op if the form was sucessfully processed, but should correctly inject the update into the case history if the form was NOT successfully processed. """
def system_ajax(request): """ Utility ajax functions for polling couch and celerymon """ type = request.GET.get('api', None) task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 12) celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None) db = XFormInstance.get_db() if type == "_active_tasks": try: tasks = filter(lambda x: x['type'] == "indexer", db.server.active_tasks()) except Unauthorized: return json_response( {'error': "Unable to access CouchDB Tasks (unauthorized)."}, status_code=500) if not is_bigcouch(): return json_response(tasks) else: # group tasks by design doc task_map = defaultdict(dict) for task in tasks: meta = task_map[task['design_document']] tasks = meta.get('tasks', []) tasks.append(task) meta['tasks'] = tasks design_docs = [] for dd, meta in task_map.items(): meta['design_document'] = dd[len('_design/'):] total_changes = sum(task['total_changes'] for task in meta['tasks']) for task in meta['tasks']: task['progress_contribution'] = task[ 'changes_done'] * 100 / total_changes design_docs.append(meta) return json_response(design_docs) elif type == "_stats": return json_response({}) elif type == "_logs": pass elif type == 'pillowtop': pillow_meta = get_all_pillows_json() supervisor_status = all_pillows_supervisor_status( [meta['name'] for meta in pillow_meta]) for meta in pillow_meta: meta.update(supervisor_status[meta['name']]) return json_response( sorted(pillow_meta, key=lambda m: m['name'].lower())) elif type == 'stale_pillows': es_index_status = [ escheck.check_case_es_index(interval=3), escheck.check_xform_es_index(interval=3), escheck.check_reportcase_es_index(interval=3), escheck.check_reportxform_es_index(interval=3) ] return json_response(es_index_status) if celery_monitoring: cresource = Resource(celery_monitoring, timeout=3) if type == "flower_poll": ret = [] try: t = cresource.get("api/tasks", params_dict={ 'limit': task_limit }).body_string() all_tasks = json.loads(t) except Exception, ex: return json_response( { 'error': "Error with getting from celery_flower: %s" % ex }, status_code=500) for task_id, traw in all_tasks.items(): # it's an array of arrays - looping through [<id>, {task_info_dict}] if 'name' in traw and traw['name']: traw['name'] = '.'.join(traw['name'].split('.')[-2:]) else: traw['name'] = None ret.append(traw) ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True) return HttpResponse(json.dumps(ret), content_type='application/json')
def soft_undelete_forms(domain, form_ids): return _soft_undelete(XFormInstance.get_db(), form_ids)
def tearDownClass(cls): CommCareCase.get_db().bulk_delete(cls.cases) XFormInstance.get_db().bulk_delete(cls.forms)
e) # one wonders if this will eventually have to paginate domains = Domain.get_all() from corehq.apps.domain.calculations import _all_domain_stats all_stats = _all_domain_stats() for dom in domains: dom.web_users = int(all_stats["web_users"][dom.name]) dom.commcare_users = int(all_stats["commcare_users"][dom.name]) dom.cases = int(all_stats["cases"][dom.name]) dom.forms = int(all_stats["forms"][dom.name]) if dom.forms: try: dom.first_submission = string_to_datetime(XFormInstance.get_db().view\ ("couchforms/all_submissions_by_domain", reduce=False, limit=1, startkey=[dom.name, "by_date"], endkey=[dom.name, "by_date", {}]).all()[0]["key"][2]).strftime("%Y-%m-%d") except Exception: dom.first_submission = "" try: dom.last_submission = string_to_datetime(XFormInstance.get_db().view\ ("couchforms/all_submissions_by_domain", reduce=False, limit=1, descending=True, startkey=[dom.name, "by_date", {}], endkey=[dom.name, "by_date"]).all()[0]["key"][2]).strftime("%Y-%m-%d") except Exception: dom.last_submission = "" else: dom.first_submission = ""
def delete_all_xforms(): # handle with care _delete_all(XFormInstance.get_db(), 'case/by_xform_id', id_func=lambda row: row['key'])
def delete_all_xforms(cls, domain=None): logger.debug("Deleting all Couch xforms for domain %s", domain) cls._delete_all(XFormInstance.get_db(), all_known_formlike_doc_types(), domain) FormProcessorTestUtils.delete_all_sql_forms(domain)
class Command(BaseCommand): args = '<id>' help = (''' Comprehensive reprocessing command for xforms that had post-submission signaling errors that may not have updated the requisite case they were attached to''') option_list = LabelCommand.option_list + \ ( make_option('--report', action='store_true', dest='do_report', default=False, help="Analyze and print a report of the data"), make_option('--from_date', action='store', dest='from_date', default="", help="Date to begin query range from"), ) def println(self, message): self.stdout.write("%s\n" % message) def printerr(self, message): self.stderr.write("%s\n" % message) def get_all_domains(self): db = Domain.get_db() return [x['key'] for x in db.view('domain/domains', reduce=False).all()] def get_all_submissions(self, domain, from_date): #/receiverwrapper/_view/all_submissions_by_domain?startkey=["tulasalud","by_date","2013-03-07"]&endkey=["tulasalud","by_date","2013-05-07"]&reduce=false db = XFormInstance.get_db() chunk = 500 start = 0 sk = [domain, "by_date", from_date.strftime("%Y-%m-%d")] ek = [domain, "by_date", (datetime.utcnow() + timedelta(days=10)).strftime("%Y-%m%d")] def call_view(sk, ek, skip, limit): return db.view('receiverwrapper/all_submissions_by_domain', startkey=sk, endkey=ek, reduce=False, limit=limit, skip=skip, include_docs=True ) view_chunk = call_view(sk, ek, start, chunk) while len(view_chunk) > 0: for item in view_chunk: if item['doc'] is not None: yield item['doc'] start += chunk view_chunk = call_view(sk, ek, start, chunk) def is_case_updated(self, submission, method="couch"): # use the same case processing utilities the case code does def _case_ids_in_couch(submission): case_view = CommCareCase.get_db().view('case/by_xform_id', key=submission['_id'], reduce=False).all() return [row['id'] for row in case_view] def _case_ids_in_es(submission): query = { "filter": { "and": [ {"term": {"xform_ids": submission['_id']}} ] }, "from": 0, "size":1 } es_results = self.es['hqcases'].post('_search', data=query) return [row['_source']['_id'] for row in es_results['hits']['hits']] \ if es_results['hits']['hits'] else [] case_blocks = extract_case_blocks(submission) case_updates = [case_update_from_block(case_block) for case_block in case_blocks] case_ids_in_form = set(cu.id for cu in case_updates) case_ids_in_db = set({ "couch": _case_ids_in_couch, "es": _case_ids_in_es, }[method](submission)) missing = case_ids_in_form - case_ids_in_db return list(case_ids_in_form), list(missing), bool(missing) def handle(self, *args, **options): self.es = get_es() try: from_date = datetime.strptime(options['from_date'], "%Y-%m-%d") except Exception, ex: self.printerr("need a valid date string --from_date YYYY-mm-dd: %s" % ex) sys.exit() self.println(','.join(HEADERS)) domains = self.get_all_domains() for ix, domain in enumerate(domains): self.printerr("Domain: %s (%d/%d)" % (domain, ix, len(domains))) for submit in self.get_all_submissions(domain, from_date): outrow = [domain, submit['received_on'], submit['doc_type'], submit['_id']] # basic case info is_dupe=False if submit['doc_type'] == 'XFormDuplicate': is_dupe=True orig_submit = XFormInstance.get_db().get(submit['form']['meta']['instanceID']) case_ids, missing, updated = self.is_case_updated(orig_submit) else: case_ids, missing, updated = self.is_case_updated(submit) if case_ids: outrow.append("|".join(case_ids)) outrow.append("|".join(missing)) outrow.append(updated) else: outrow.append("nocase") outrow.append("|".join(missing)) # would be weird if something was here outrow.append("no update") def _should_write(): # if we want to make this more configurable can adjust # for now only output if there's a missing case and if # it's not a dupe/deprecated return missing and submit['doc_type'] not in ['XFormDeprecated', 'XFormDuplicate'] if _should_write(): self.println(','.join(str(x) for x in outrow))
def __init__(self, **kwargs): super(UnknownUsersPillow, self).__init__(**kwargs) self.couch_db = XFormInstance.get_db() self.user_db = CouchUser.get_db() self.es = get_es()
def setUp(self): super(ChangeFeedDbTest, self).setUp() self.couch_db = XFormInstance.get_db() self.update_seq = get_current_seq(self.couch_db)
def print_stats(self, domain, short=True, diffs_only=False): db = get_diff_db(domain) try: diff_stats = db.get_diff_stats() except OperationalError: diff_stats = {} has_diffs = False for doc_type in doc_types(): form_ids_in_couch = set(get_form_ids_by_type(domain, doc_type)) form_ids_in_sql = set( FormAccessorSQL.get_form_ids_in_domain_by_type( domain, doc_type)) diff_count, num_docs_with_diffs = diff_stats.pop(doc_type, (0, 0)) has_diffs |= self._print_status(doc_type, form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) form_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "XFormInstance-Deleted", XFormInstance.get_db())) form_ids_in_sql = set( FormAccessorSQL.get_deleted_form_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop( "XFormInstance-Deleted", (0, 0)) has_diffs |= self._print_status("XFormInstance-Deleted", form_ids_in_couch, form_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) case_ids_in_couch = set(get_case_ids_in_domain(domain)) case_ids_in_sql = set(CaseAccessorSQL.get_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop( "CommCareCase", (0, 0)) has_diffs |= self._print_status('CommCareCase', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) case_ids_in_couch = set( get_doc_ids_in_domain_by_type(domain, "CommCareCase-Deleted", XFormInstance.get_db())) case_ids_in_sql = set( CaseAccessorSQL.get_deleted_case_ids_in_domain(domain)) diff_count, num_docs_with_diffs = diff_stats.pop( "CommCareCase-Deleted", (0, 0)) has_diffs |= self._print_status('CommCareCase-Deleted', case_ids_in_couch, case_ids_in_sql, diff_count, num_docs_with_diffs, short, diffs_only) if diff_stats: for key, counts in diff_stats.items(): diff_count, num_docs_with_diffs = counts has_diffs |= self._print_status(key, set(), set(), diff_count, num_docs_with_diffs, short, diffs_only) if diffs_only and not has_diffs: print( shell_green("No differences found between old and new docs!")) return has_diffs
def tearDown(self): try: XFormInstance.get_db().delete_doc("7H46J37FGH3") except: pass
def soft_delete_forms(domain, form_ids, deletion_date=None, deletion_id=None): return _soft_delete(XFormInstance.get_db(), form_ids, deletion_date, deletion_id)
def get_all_forms_in_all_domains(): return [ XFormInstance.wrap(doc) for doc in get_all_docs_with_doc_types( XFormInstance.get_db(), ['XFormInstance']) ]
def assign_new_id(cls, xform): assert not xform.persistent_blobs, "some blobs would be lost" new_id = XFormInstance.get_db().server.next_uuid() xform._id = new_id return xform
def tearDown(self): view_kwargs = { 'startkey': ['submission user', TEST_DOMAIN, self.user1._id], 'endkey': ['submission user', TEST_DOMAIN, self.user1._id, {}], } FormProcessorTestUtils._delete_all_from_view(XFormInstance.get_db(), 'all_forms/view', view_kwargs)
def get_doc_count(model_class, where, entity, domain): sql_estimate = estimate_partitioned_row_count(model_class, where) couchdb = XFormInstance.get_db() couch_count = get_couch_doc_count(domain, entity, couchdb) return min(sql_estimate, couch_count)
def process_indicators_for_case(namespaces, domain, doc_dict): case_type = doc_dict.get('type') if not case_type: return case_indicator_defs = [] for namespace in namespaces: case_indicator_defs.extend( CaseIndicatorDefinition.get_all(namespace, domain, case_type=case_type)) try: indicator_case = IndicatorCase.wrap_for_indicator_db(doc_dict) indicator_case.update_indicators_in_bulk(case_indicator_defs, logger=pillow_logging, save_on_update=False) indicator_case.save() except Exception as e: pillow_logging.error( "Error creating for MVP Indicator for form %(form_id)s: " "%(error)s" % { 'form_id': doc_dict['_id'], 'error': e, }, ) raise # Now Update Data From Case to All Related Xforms (ewwww) xform_ids = doc_dict.get('xform_ids', []) if not xform_ids: return for xform_id in xform_ids: related_xform_indicators = [] try: # first try to get the doc from the indicator DB xform_doc = IndicatorXForm.get(xform_id) except ResourceNotFound: # if that fails fall back to the main DB try: xform_dict = XFormInstance.get_db().get(xform_id) xform_doc = IndicatorXForm.wrap_for_indicator_db(xform_dict) related_xform_indicators = get_form_indicators( namespaces, domain, xform_doc.xmlns) except ResourceNotFound: pillow_logging.error( "Could not find an XFormInstance with id %(xform_id)s " "related to Case %(case_id)s" % { 'xform_id': xform_id, 'case_id': doc_dict['_id'], }) continue if not xform_doc.xmlns: continue for namespace in namespaces: related_xform_indicators.extend( CaseDataInFormIndicatorDefinition.get_all( namespace, domain, xmlns=xform_doc.xmlns)) xform_doc.update_indicators_in_bulk(related_xform_indicators, logger=pillow_logging, save_on_update=False) xform_doc.save()
# fix the XFormInstance instance.form['location_id'] = case.location_id # fix the actual form.xml xml_object = etree.fromstring(instance.get_xml()) location_id_node = xml_object.find( re.sub('}.*', '}location_id', xml_object.tag)) location_id_node.text = case.location_id updated_xml = etree.tostring(xml_object) attachment_builder = CouchAttachmentsBuilder( instance._attachments) attachment_builder.add( name='form.xml', content=updated_xml, content_type=instance._attachments['form.xml'] ['content_type']) instance._attachments = attachment_builder.to_json() print 'Updating XFormInstance:', doc['_id'] to_save.append(instance) except Exception: print 'Failed to save XFormInstance:', doc['_id'] if len(to_save) > 500: XFormInstance.get_db().bulk_save(to_save) to_save = [] if to_save: XFormInstance.get_db().bulk_save(to_save)
def reprocess_xform_error(form): """ Attempt to re-process an error form. This was created specifically to address the issue of out of order forms and child cases (form creates child case before parent case has been created). See http://manage.dimagi.com/default.asp?250459 :param form_id: ID of the error form to process """ from corehq.form_processor.interfaces.processor import FormProcessorInterface from corehq.form_processor.submission_post import SubmissionPost from corehq.form_processor.utils import should_use_sql_backend from corehq.form_processor.backends.sql.dbaccessors import CaseAccessorSQL, FormAccessorSQL, LedgerAccessorSQL from corehq.blobs.mixin import bulk_atomic_blobs from couchforms.models import XFormInstance from casexml.apps.case.signals import case_post_save from corehq.form_processor.interfaces.processor import ProcessedForms from corehq.form_processor.backends.sql.processor import FormProcessorSQL if not form: raise Exception('Form with ID {} not found'.format(form.form_id)) if not form.is_error: raise Exception('Form was not an error form: {}={}'.format(form.form_id, form.doc_type)) # reset form state prior to processing if should_use_sql_backend(form.domain): form.state = XFormInstanceSQL.NORMAL else: form.doc_type = 'XFormInstance' form.initial_processing_complete = True form.problem = None cache = FormProcessorInterface(form.domain).casedb_cache( domain=form.domain, lock=True, deleted_ok=True, xforms=[form] ) with cache as casedb: case_stock_result = SubmissionPost.process_xforms_for_cases([form], casedb) if case_stock_result: stock_result = case_stock_result.stock_result if stock_result: assert stock_result.populated cases = case_stock_result.case_models if should_use_sql_backend(form.domain): for case in cases: CaseAccessorSQL.save_case(case) if stock_result: LedgerAccessorSQL.save_ledger_values(stock_result.models_to_save) FormAccessorSQL.update_form_problem_and_state(form) FormProcessorSQL._publish_changes( ProcessedForms(form, None), cases, stock_result ) else: with bulk_atomic_blobs([form] + cases): XFormInstance.save(form) # use this save to that we don't overwrite the doc_type XFormInstance.get_db().bulk_save(cases) if stock_result: stock_result.commit() case_stock_result.stock_result.finalize() case_stock_result.case_result.commit_dirtiness_flags() for case in cases: case_post_save.send(case.__class__, case=case) return form
for ledger in ledgers: if ledger.ledger_reference in ledgers_updated: logger.info('Rebuilding ledger: %s', ledger.ledger_reference) if save: # only rebuild upated ledgers interface.ledger_processor.hard_rebuild_ledgers( **ledger.ledger_reference._asdict()) else: if save: with bulk_atomic_blobs([form] + cases): XFormInstance.save( form ) # use this save to that we don't overwrite the doc_type XFormInstance.get_db().bulk_save(cases) stock_result.commit() save and case_stock_result.stock_result.finalize() save and case_stock_result.case_result.commit_dirtiness_flags() return ReprocessingResult(form, cases, ledgers) def _log_changes(slug, cases, stock_updates, stock_deletes): if logger.isEnabledFor(logging.INFO): case_ids = [case.case_id for case in cases] logger.info( "%s changes:\n\tcases: %s\n\tstock changes%s\n\tstock deletes%s", slug, case_ids, stock_updates, stock_deletes)
def soft_undelete_forms(domain, form_ids): def _form_undelete(doc): doc['server_modified_on'] = json_format_datetime(datetime.utcnow()) return _soft_undelete(XFormInstance.get_db(), form_ids, _form_undelete)
def commit_plan(domain, planning_db): assert get_planning_db_filepath(domain) == planning_db.db_filepath for form in planning_db.get_forms(): XFormInstance.get_db().save_doc(form) for case in planning_db.get_cases(): CommCareCase.get_db().save_doc(case)
def save(self, **kwargs): self.doc_type = 'IndicatorXForm' assert self.get_db().uri != XFormInstance.get_db().uri super(IndicatorXForm, self).save(**kwargs)