def close_or_update_investigation_cases(self, all_cases, retain_case_id, episode_case_id, investigation_interval, updates): all_case_ids = [ investigation_case.case_id for investigation_case in all_cases ] # ToDo: refetch investigation_cases in case the len of set is different from list # remove duplicates in case ids to remove so that we don't retain and close # the same case by mistake all_case_ids = set(all_case_ids) case_ids_to_close = all_case_ids.copy() case_ids_to_close.remove(retain_case_id) for investigation_case in all_cases: self.writerow({ "episode_case_id": episode_case_id, "investigation_interval": investigation_interval, "investigation_case_id": investigation_case.case_id, "modified_on": investigation_case.get_case_property(DATE_MODIFIED_FIELD), "updates": updates, "update/close": ('update' if investigation_case.case_id == retain_case_id else 'closed') }) if self.commit: updates = [(case_id, { 'close_reason': "duplicate_reconciliation" }, True) for case_id in case_ids_to_close] bulk_update_cases(DOMAIN, updates, self.__module__)
def close_cases(self, all_cases, occurrence_case_id, retain_case): # remove duplicates in case ids to remove so that we dont retain and close # the same case by mistake all_case_ids = set([case.case_id for case in all_cases]) retain_case_id = retain_case.case_id case_ids_to_close = all_case_ids.copy() case_ids_to_close.remove(retain_case_id) case_accessor = CaseAccessors(DOMAIN) closing_extension_case_ids = case_accessor.get_extension_case_ids( case_ids_to_close) self.writerow({ "occurrence_case_id": occurrence_case_id, "retain_case_id": retain_case_id, "closed_case_ids": ','.join(map(str, case_ids_to_close)), "closed_extension_case_ids": ','.join(map(str, closing_extension_case_ids)), "person_case_version": self.person_case.get_case_property('case_version'), "person_case_dataset": self.person_case.get_case_property('dataset') }) if self.commit: updates = [(case_id, { 'referral_closed_reason': "duplicate_reconciliation", 'referral_closed_date': datetime.datetime.now(pytz.timezone(ENIKSHAY_TIMEZONE)).date() }, True) for case_id in case_ids_to_close] bulk_update_cases(DOMAIN, updates, self.__module__)
def handle(self, domain, **options): batch_size = 100 updates = [] errors = [] case_ids = get_all_episode_ids(domain) cases = iter_all_active_person_episode_cases(domain, case_ids) for person, episode in with_progress_bar(cases, len(case_ids), oneline=False): try: update_json = self.updater(domain, person, episode).update_json() except Exception as e: errors.append( [person.case_id, episode.case_id, episode.domain, e]) continue if update_json: updates.append((episode.case_id, update_json, False)) if len(updates) >= batch_size: if options['commit']: bulk_update_cases(domain, updates, self.__module__) updates = [] if len(updates) > 0: if options['commit']: bulk_update_cases(domain, updates, self.__module__) self.write_errors(errors)
def run_batch(self, case_ids): """Run all case updaters against the case_ids passed in """ device_id = "%s.%s" % (__name__, type(self).__name__) update_count = 0 noupdate_count = 0 error_count = 0 success_count = 0 case_batches = 0 ledger_batches = 0 errors = [] with Timer() as t: batch_size = 100 case_updates = [] ledger_updates = [] for episode in self._get_open_episode_cases(case_ids): did_error = False update_json = {} for updater in self.updaters: try: _updater = updater(self.domain, episode) case_update = _updater.update_json() if hasattr(_updater, 'ledger_updates'): ledger_updates.extend(_updater.ledger_updates()) update_json.update(get_updated_fields(episode.dynamic_case_properties(), case_update)) except Exception as e: did_error = True error = [episode.case_id, episode.domain, updater.__name__, e] errors.append(error) logger.error(error) if did_error: error_count += 1 else: success_count += 1 if update_json: case_updates.append((episode.case_id, update_json, False)) update_count += 1 else: noupdate_count += 1 if len(case_updates) >= batch_size: bulk_update_cases(self.domain, case_updates, device_id) case_updates = [] case_batches += 1 if len(ledger_updates) >= batch_size: bulk_update_ledger_cases(self.domain, ledger_updates) ledger_updates = [] ledger_batches += 1 if len(case_updates) > 0: bulk_update_cases(self.domain, case_updates, device_id) if len(ledger_updates) > 0: bulk_update_ledger_cases(self.domain, ledger_updates) return BatchStatus(update_count, noupdate_count, success_count, errors, case_batches, ledger_batches, t.interval)
def update_vouchers(self, voucher_updates): print "updating voucher cases" for chunk in chunked(with_progress_bar(voucher_updates), 100): updates = [ (update.case_id, update.properties, False) for update in chunk ] if self.commit: bulk_update_cases(self.domain, updates, self.__module__)
def payment_confirmation(request, domain): try: updates = _get_case_updates(request, domain) updates = _validate_updates_exist(domain, updates) except ApiError as e: if not settings.UNIT_TESTING: notify_exception(request, "BETS sent the eNikshay API a bad request.") return json_response({"error": e.message}, status_code=e.status_code) bulk_update_cases(domain, [ (update.case_id, update.properties, False) for update in updates ], __name__ + ".payment_confirmation") return json_response({'status': SUCCESS})
def process_payment_confirmations(domain, payment_confirmations): for chunk in chunked(payment_confirmations, 100): try: bulk_update_cases( domain, [(update.case_id, update.properties, False) for update in chunk], "custom.enikshay.integrations.bets.views.payment_confirmation") except Exception as e: notify_exception( request=None, message=PAYMENT_CONFIRMATION_FAILURE, details=json.dumps([update.to_json() for update in chunk]), )
def handle(self, domain, **options): self.domain = domain self.accessor = CaseAccessors(domain) commit = options['commit'] self.id_generator = ReadableIdGenerator(domain, commit) filename = '{}-{}.csv'.format( self.__module__.split('.')[-1], datetime.datetime.now().strftime('%Y-%m-%d_%H.%M.%S')) print("Logging actions to {}".format(filename)) with open(filename, 'w') as f: logfile = csv.DictWriter(f, self.logfile_fields, extrasaction='ignore') logfile.writeheader() print("Finding duplicates") bad_case_stubs = get_duplicated_case_stubs(self.domain, CASE_TYPE_PERSON) bad_cases = self.accessor.iter_cases(stub['case_id'] for stub in bad_case_stubs) print("Processing duplicate cases") for person_case in with_progress_bar(bad_cases, len(bad_case_stubs)): if person_case.get_case_property( 'enrolled_in_private') == 'true': updates = list( filter(None, self.get_private_updates(person_case))) else: updates = list( filter(None, self.get_public_updates(person_case))) person_info = self.get_person_case_info(person_case) for case, update in updates: log = { unidecode(k): unidecode(v) for d in [person_info, update] for k, v in d.items() if v } log['case_type'] = case.type log['case_id'] = case.case_id logfile.writerow(log) if commit: update_tuples = [(case.case_id, update, False) for case, update in updates] bulk_update_cases(self.domain, update_tuples, self.__module__)
def handle(self, domain, infile, logfile, *args, **options): self.domain = domain self.case_accessor = CaseAccessors(self.domain) with open(infile, 'r', encoding='utf-8') as f, open(logfile, 'w', encoding='utf-8') as log: reader = csv.reader(f) _, case_prop_name = next(reader) log.write('--------Successful Form Ids----------\n') failed_updates = [] for rows in chunked(reader, 100): updates = [(case_id, { case_prop_name: prop }, False) for case_id, prop in rows] try: xform, cases = bulk_update_cases(self.domain, updates, self.__module__) log.write(xform.form_id + '\n') except Exception as e: print('error') print(str(e)) failed_updates.extend(u[0] for u in updates) log.write('--------Failed Cases--------------\n') for case_id in failed_updates: log.write(case_id + '\n') log.write('--------Logging Complete--------------\n') print('-------------COMPLETE--------------')
def handle(self, shard, log_file, **options): self.domain = 'icds-cas' self.db = shard self.case_accessor = CaseAccessors(self.domain) failed_updates = [] with open(log_file, "w", encoding='utf-8') as fh: fh.write('--------Successful Form Ids----------\n') chunk_num = 1 for orphan_case_chunk in self._get_cases(): print('Currently on chunk {}'.format(chunk_num)) case_tupes = [(case_id, {}, True) for case_id in orphan_case_chunk] try: xform, cases = bulk_update_cases( self.domain, case_tupes, self.__module__) fh.write(xform.form_id + '\n') except LocalSubmissionError as e: print('submission error') print(six.text_type(e)) failed_updates.extend(orphan_case_chunk) except Exception as e: print('unexpected error') print(six.text_type(e)) failed_updates.extend(orphan_case_chunk) chunk_num += 1 fh.write('--------Failed Cases--------------\n') for case_id in failed_updates: fh.write(case_id + '\n') fh.write('--------Logging Complete--------------\n') print('-------------COMPLETE--------------')
def handle(self, domain, log_file, **options): total_cases = CaseES().domain(domain).case_type('household').is_closed().count() self.case_accessor = CaseAccessors(domain) failed_updates = [] with open(log_file, "w", encoding='utf-8') as fh: fh.write('--------Successful Form Ids----------\n') for cases in chunked(with_progress_bar(self._get_cases_to_process(domain), total_cases), 100): related_cases = self._get_related_cases(cases) case_tupes = [(case_id, {}, True) for case_id in related_cases] try: xform, cases = bulk_update_cases( domain, case_tupes, self.__module__) fh.write(xform.form_id + '\n') except LocalSubmissionError as e: print('submission error') print(six.text_type(e)) failed_updates.extend(related_cases) except Exception as e: print('unexpected error') print(six.text_type(e)) failed_updates.extend(related_cases) fh.write('--------Failed Cases--------------\n') for case_id in failed_updates: fh.write(case_id) print('-------------COMPLETE--------------')
def update_cases(self): sector = get_sector(self._person_case) case_updates = [] for prop, value in six.iteritems(self.request_json): try: param = self.api_spec.get_param(prop, sector) except KeyError: raise NinetyNineDotsException( "{} is not a valid parameter to update".format(prop)) if not param.direction & DIRECTION_INBOUND: raise NinetyNineDotsException( "{} is not a valid parameter to update".format(prop)) case_type = param.get_by_sector('case_type', sector) case_id = self.case_types_to_cases[case_type].case_id if param.setter: update = to_function(param.setter)(param, value, sector) else: update = {param.get_by_sector('case_property', sector): value} case_updates.append((case_id, update, False)) return bulk_update_cases( self.domain, case_updates, "{}.{}".format(self.__module__, self.__class__.__name__), )
def handle(self, shard, log_file, **options): self.domain = 'icds-cas' self.db = shard self.case_accessor = CaseAccessors(self.domain) failed_updates = [] with open(log_file, "w") as fh: fh.write('--------Successful Form Ids----------\n') chunk_num = 1 for orphan_case_chunk in self._get_cases(): print('Currently on chunk {}'.format(chunk_num)) case_tupes = [(case_id, {}, True) for case_id in orphan_case_chunk] try: xform, cases = bulk_update_cases( self.domain, case_tupes, self.__module__) fh.write(xform.form_id + '\n') except LocalSubmissionError as e: print('submission error') print(unicode(e)) failed_updates.extend(orphan_case_chunk) except Exception as e: print('unexpected error') print(unicode(e)) failed_updates.extend(orphan_case_chunk) chunk_num += 1 fh.write('--------Failed Cases--------------\n') for case_id in failed_updates: fh.write(case_id + '\n') fh.write('--------Logging Complete--------------\n') print('-------------COMPLETE--------------')
def handle(self, log_file, **options): self.domain = 'hki-nepal-suaahara-2' loc_mapping = {} locs = SQLLocation.objects.filter(domain=self.domain, level=4) for loc in locs: loc_mapping[loc.site_code] = loc.location_id failed_updates = [] household_cases = CaseES().domain(self.domain).case_type('household').count() member_cases = CaseES().domain(self.domain).case_type('household_member').count() total_cases = household_cases + member_cases with open(log_file, "w", encoding='utf-8') as fh: fh.write('--------Successful Form Ids----------') for cases in chunked(with_progress_bar(self._get_cases_to_process(), total_cases), 100): cases_to_update = self._process_cases(cases, failed_updates, loc_mapping) try: xform, cases = bulk_update_cases( self.domain, cases_to_update, self.__module__) fh.write(xform.form_id) except LocalSubmissionError as e: print(six.text_type(e)) failed_updates.extend(case[0] for case in cases_to_update) fh.write('--------Failed Cases--------------') for case_id in failed_updates: fh.write(case_id)
def handle(self, domain, log_file, **options): total_cases = CaseES().domain(domain).case_type( 'household').is_closed().count() self.case_accessor = CaseAccessors(domain) failed_updates = [] with open(log_file, "w", encoding='utf-8') as fh: fh.write('--------Successful Form Ids----------\n') for cases in chunked( with_progress_bar(self._get_cases_to_process(domain), total_cases), 100): related_cases = self._get_related_cases(cases) case_tupes = [(case_id, {}, True) for case_id in related_cases] try: xform, cases = bulk_update_cases(domain, case_tupes, self.__module__) fh.write(xform.form_id + '\n') except LocalSubmissionError as e: print('submission error') print(six.text_type(e)) failed_updates.extend(related_cases) except Exception as e: print('unexpected error') print(six.text_type(e)) failed_updates.extend(related_cases) fh.write('--------Failed Cases--------------\n') for case_id in failed_updates: fh.write(case_id) print('-------------COMPLETE--------------')
def close_cases(self, all_cases, retain_case, associated_case_id, reconciling_case_type): # remove duplicates in case ids to remove so that we don't retain and close # the same case by mistake all_case_ids = set([case.case_id for case in all_cases]) retain_case_id = retain_case.case_id case_ids_to_close = all_case_ids.copy() case_ids_to_close.remove(retain_case_id) case_accessor = CaseAccessors(DOMAIN) closing_extension_case_ids = case_accessor.get_extension_case_ids( case_ids_to_close) self.writerow({ "case_type": reconciling_case_type, "associated_case_id": associated_case_id, "retain_case_id": retain_case_id, "closed_case_ids": ','.join(map(str, case_ids_to_close)), "closed_extension_case_ids": ','.join(map(str, closing_extension_case_ids)), "retained_case_date_opened": str(retain_case.opened_on), "retained_case_episode_type": retain_case.get_case_property("episode_type"), "retained_case_is_active": retain_case.get_case_property("is_active"), "closed_cases_details": ({ a_case.case_id: { "last_modified_at(utc)": str(last_user_edit_at(a_case)), "episode_type": a_case.get_case_property("episode_type"), "is_active": a_case.get_case_property("is_active") } for a_case in all_cases if a_case.case_id != retain_case_id }) }) if self.commit: updates = [(case_id, { 'close_reason': "duplicate_reconciliation" }, True) for case_id in case_ids_to_close] bulk_update_cases(DOMAIN, updates, self.__module__)
def handle(self, domain, case_type, *args, **options): perform_update = True query = (CaseES(es_instance_alias=ES_EXPORT_INSTANCE).domain( domain).case_type(case_type).is_closed(False).term( 'name.exact', '')) cases_count = query.count() print("Number of cases to be updated approximately: %s" % cases_count) if not input("Do you wish to update cases (y/n)") == 'y': perform_update = False if not input("Do you wish to just log updates (y/n)") == 'y': exit(0) case_ids = query.get_ids() print("Begin iterating %s cases" % len(case_ids)) case_accessor = CaseAccessors(domain) case_updates = [] filename = "case_updates_%s_%s_%s.csv" % (domain, case_type, datetime.utcnow()) with open(filename, 'w') as f: writer = csv.DictWriter(f, ['case_id', 'new_value']) writer.writeheader() for case_id in with_progress_bar(case_ids): case = case_accessor.get_case(case_id) if case.name: continue update_to_name = get_last_non_blank_value(case, 'name') if update_to_name: writer.writerow({ 'case_id': case_id, 'new_value': update_to_name }) if perform_update: case_updates.append((case_id, { 'name': update_to_name }, False)) # update batch when we have the threshold if len(case_updates) == CASE_UPDATE_BATCH: bulk_update_cases(domain, case_updates, DEVICE_ID) case_updates = [] # submit left over case updates if case_updates: print("Performing last batch of updates") bulk_update_cases(domain, case_updates, DEVICE_ID) print("Finished. Update details in %s" % filename)
def run(self): # iterate over all open 'episode' cases and set 'adherence' properties update_count = 0 noupdate_count = 0 error_count = 0 with Timer() as t: batch_size = 100 updates = [] for episode in self._get_open_episode_cases(): adherence_update = EpisodeAdherenceUpdate(episode, self) voucher_update = EpisodeVoucherUpdate(self.domain, episode) test_update = EpisodeTestUpdate(self.domain, episode) episode_facility_id_migration = EpisodeFacilityIDMigration(self.domain, episode) try: update_json = adherence_update.update_json() update_json.update(voucher_update.update_json()) update_json.update(test_update.update_json()) update_json.update(episode_facility_id_migration.update_json()) if update_json: updates.append((episode.case_id, update_json, False)) update_count += 1 else: noupdate_count += 1 if len(updates) == batch_size: bulk_update_cases(self.domain, updates) updates = [] except Exception, e: error_count += 1 logger.error( "Error calculating updates for episode case_id({}): {}".format( episode.case_id, e ) ) if len(updates) > 0: bulk_update_cases(self.domain, updates)
def handle(self, domain, infile, logfile, *args, **options): self.domain = domain self.case_accessor = CaseAccessors(self.domain) with open(infile, 'r', encoding='utf-8') as f, open(logfile, 'w', encoding='utf-8') as log: reader = csv.reader(f) _, case_prop_name = next(reader) log.write('--------Successful Form Ids----------\n') failed_updates = [] for rows in chunked(reader, 100): updates = [(case_id, {case_prop_name: prop}, False) for case_id, prop in rows] try: xform, cases = bulk_update_cases( self.domain, updates, self.__module__) log.write(xform.form_id + '\n') except Exception as e: print('error') print(six.text_type(e)) failed_updates.extend(u[0] for u in updates) log.write('--------Failed Cases--------------\n') for case_id in failed_updates: log.write(case_id + '\n') log.write('--------Logging Complete--------------\n') print('-------------COMPLETE--------------')
# remove duplicates in case ids to remove so that we dont retain and close # the same case by mistake all_case_ids = {case.case_id for case in all_cases} retain_case_id = retain_case.case_id case_ids_to_close = all_case_ids.copy() case_ids_to_close.remove(retain_case_id) case_accessor = CaseAccessors(DOMAIN) closing_extension_case_ids = case_accessor.get_extension_case_ids(case_ids_to_close) self.writerow({ "occurrence_case_id": occurrence_case_id, "person_case_id": self.person_case_id, "drug_id": drug_id, "retain_case_id": retain_case_id, "retain_reason": retain_reason, "closed_case_ids": ','.join(map(str, case_ids_to_close)), "closed_extension_case_ids": ','.join(map(str, closing_extension_case_ids)) }) if self.commit: updates = [(case_id, {'close_reason': "duplicate_reconciliation"}, True) for case_id in case_ids_to_close] bulk_update_cases(DOMAIN, updates, self.__module__) def get_open_drug_resistance_cases_from_occurrence(occurrence_case_id): case_accessor = CaseAccessors(DOMAIN) all_cases = case_accessor.get_reverse_indexed_cases([occurrence_case_id]) return [case for case in all_cases if not case.closed and case.type == CASE_TYPE_DRUG_RESISTANCE]
def handle(self, domain, **options): commit = options['commit'] filename = "reassign_from_facility-{}.csv".format( datetime.datetime.now().strftime('%Y-%m-%d_%H.%M.%S')) columns = [ 'case_id', 'facility_assigned_to', 'owner_id', 'last_owner_id_changed', 'last_facility_assigned_to_changed', 'note' ] case_ids = get_all_episode_ids(domain) cases = iter_all_active_person_episode_cases(domain, case_ids, sector='private') bad_cases = [] to_update = [] for person, _ in with_progress_bar(cases, length=len(case_ids)): facility_assigned_to = person.get_case_property( 'facility_assigned_to') owner_id = person.owner_id if facility_assigned_to == owner_id: continue if not facility_assigned_to and owner_id in [MJK, ALERT_INDIA]: # cases with a blank facility and owned by MJK or Alert-India are known about already continue owner_id_changes = sorted(get_all_changes_to_case_property( person, 'owner_id'), key=lambda c: c.modified_on, reverse=True) facility_id_changes = sorted(get_all_changes_to_case_property( person, 'facility_assigned_to'), key=lambda c: c.modified_on, reverse=True) case_dict = { 'case_id': person.case_id, 'facility_assigned_to': facility_assigned_to, 'owner_id': owner_id, } try: case_dict['last_owner_id_changed'] = owner_id_changes[ 0].modified_on case_dict[ 'last_facility_assigned_to_changed'] = facility_id_changes[ 0].modified_on if owner_id_changes[0].modified_on < facility_id_changes[ 0].modified_on: case_dict['note'] = 'updated' to_update.append((person.case_id, { "owner_id": facility_assigned_to }, False)) else: case_dict['note'] = 'not updated' except IndexError as e: case_dict['last_owner_id_changed'] = None case_dict['last_facility_assigned_to_changed'] = None case_dict['note'] = 'no changes found: {}'.format( six.text_type(e)) bad_cases.append(case_dict) if commit: print("Updating: ", len(to_update), " cases") for update in chunked(to_update, 100): bulk_update_cases(domain, update, self.__module__) else: print("Would have updated: ", len(to_update), " cases") with open(filename, 'w') as f: writer = csv.DictWriter(f, fieldnames=columns) writer.writeheader() for case in bad_cases: writer.writerow(case)