def render_map(request, share_token): entity_preference = get_entity_preference_by_share_token( get_db_manager("public"), share_token) dbm = get_db_manager(entity_preference.org_id) if not _flag_active("idnr_map", entity_preference.org_id.split('_')[-1].upper()): raise Http404 form_model = get_form_model_by_entity_type( dbm, [entity_preference.entity_type.lower()]) return render_to_response('map.html', { "entity_type": entity_preference.entity_type, "filters": _get_filters(form_model, entity_preference.filters), "geo_jsons": geo_jsons(dbm, entity_preference.entity_type, request.GET, entity_preference.details, entity_preference.specials), "fallback_location": entity_preference.fallback_location, "mapbox_api_key": get_mapbox_api_key(request.META['HTTP_HOST']) }, context_instance=RequestContext(request))
def create_questionnaire_templates(): db_name = settings.QUESTIONNAIRE_TEMPLATE_DB_NAME existing_dbm = get_db_manager(db_name) _delete_db_and_remove_db_manager(existing_dbm) recreated_dbm = get_db_manager(db_name) _create_view(recreated_dbm) file = settings.QUESTIONNAIRE_TEMPLATE_JSON_DATA_FILE docs = create_template_from_json_file(recreated_dbm, file) return docs
def handle(self, *args, **options): get_cache_manager().flush_all() for database_name in all_db_names(): print("Database %s") % (database_name, ) print 'Deleting...' manager = get_db_manager(database_name) _delete_db_and_remove_db_manager(manager) for database_name in document_stores_to_process(args): recreated_manager = get_db_manager(database_name) print "Syncing Views....." SyncOnlyChangedViews().sync_view(recreated_manager) print "Loading data....." load_data() print "Done."
def recreate_index_for_db(database_name): start = time.time() es = get_elasticsearch_handle() logger = logging.getLogger(database_name) try: es.delete_index(database_name) except Exception as e: logger.info("Could not delete index " + str(e.message)) response = es.create_index(database_name, settings={ "number_of_shards": 1, "number_of_replicas": 0 }) logger.info('%s search index created : %s' % (database_name, response.get('ok'))) dbm = get_db_manager(database_name) try: create_all_mappings(dbm) create_all_indices(dbm) except Exception as e: logger.exception("recreate index failed for database %s for" % dbm.database_name) logger.info( 'Time taken (seconds) for indexing {database_name} : {timetaken}'. format(database_name=database_name, timetaken=(time.time() - start)))
def send_reminders_scheduled_on(on_date, sms_client): """ Sends out reminders scheduled for the given date, for each organization. """ assert isinstance(on_date, date) try: logger.info("Sending reminders for date:- %s" % on_date) paid_organization = _get_active_paid_organizations() for org in paid_organization: try: logger.info("Organization %s" % org.name) org_setting = OrganizationSetting.objects.filter( organization=org)[0] smsc = org_setting.outgoing_number.smsc charged_sms = smsc.vumi_username in settings.SMSC_WITHOUT_STATUS_REPORT manager = get_db_manager(org_setting.document_store) from_number = org_setting.sms_tel_number.split(',')[0] send_reminders_for_an_organization(org, on_date, sms_client, from_number=from_number, dbm=manager, charged_sms=charged_sms) logger.info("Successfully sent reminders for Org: %s.", org.name) except Exception as e: logger.exception( "Error while sending reminders for organization : %s" % org.name) logger.info("Done sending all reminders.") except Exception as e: logger.exception("Exception while sending reminders")
def check_project_and_formmodel_name_mismatch(db_name): try: logging.info('Starting checking for database %s' % db_name) dbm = get_db_manager(db_name) check_for_name_mismatch(dbm) except Exception as e: logging.exception(e.message)
def update_associated_submissions(database_name, old_form_code, new_form_code, deleted_question_codes): try: manager = get_db_manager(database_name) update_submissions_for_form_code_change(manager, new_form_code, old_form_code) update_submissions_for_form_field_change(manager, old_form_code, deleted_question_codes) except Exception as e: current.retry(exc=e)
def create_search_indices_for_deleted_datasender(db_name): logger = logging.getLogger(db_name) try: logger.info('Starting indexing') dbm = get_db_manager(db_name) es = get_elasticsearch_handle(timeout=600) form_model = get_form_model_by_code(dbm, REGISTRATION_FORM_CODE) datasenders = [] for entity in get_all_entities_include_voided(dbm, REPORTER_ENTITY_TYPE): if not entity.data: continue if entity.is_void() or entity.short_code == 'test': datasender_dict = _create_datasender_dict( dbm, entity, REPORTER, form_model) datasender_dict.update({'id': entity.id}) datasenders.append(datasender_dict) if datasenders: es.bulk_index(dbm.database_name, REPORTER, datasenders) logger.info('Created index for datasenders with ids :' + str([a.get('id') for a in datasenders])) logger.info('Completed Indexing') mark_as_successful(db_name) except Exception as e: logger.exception(e.message)
def migration_to_add_email_data_for_web_users_in_couch(db_name): logger = logging.getLogger(db_name) logger.info('Starting Migration') mark_as_completed(db_name) manager = get_db_manager(db_name) try: form_model = get_form_model_by_code(manager, REGISTRATION_FORM_CODE) except FormModelDoesNotExistsException as f: logger.warning(f.message) return email_field = TextField(name=EMAIL_FIELD, code=EMAIL_FIELD, label="What is the subject's email", defaultValue="", instruction="Enter email id", constraints=[TextLengthConstraint(max=50)], required=False) try: form_model.add_field(email_field) form_model.save() logger.info("Migrated registration form") except QuestionCodeAlreadyExistsException as e: logger.warning('email field is present' + e.message) except Exception as e: logger.exception(e.message)
def migration_to_add_email_data_for_web_users_in_couch(db_name): logger = logging.getLogger(db_name) logger.info('Starting Migration') mark_as_completed(db_name) manager = get_db_manager(db_name) email_field_code = "email" try: form_model = get_form_model_by_code(manager, REGISTRATION_FORM_CODE) except FormModelDoesNotExistsException as f: logger.warning(f.message) return email_field_label = form_model._get_field_by_code(email_field_code).name email_ddtype = form_model._get_field_by_code(email_field_code).ddtype org_id = OrganizationSetting.objects.get( document_store=manager.database_name).organization_id user_profiles = NGOUserProfile.objects.filter(org_id=org_id) for user_profile in user_profiles: try: short_code = user_profile.reporter_id email_value = user_profile.user.email data = (email_field_label, email_value, email_ddtype) if short_code: add_email_data_to_entity_document(manager, short_code, data, logger) except DataObjectAlreadyExists as e: logger.warning(e.message) except Exception as e: logger.exception("FAILED to migrate: %s" % short_code)
def send_reminders_scheduled_on(on_date, sms_client): """ Sends out reminders scheduled for the given date, for each organization. """ assert isinstance(on_date, date) try: logger.info("Sending reminders for date:- %s" % on_date) paid_organization = _get_active_paid_organization() for org in paid_organization: try: logger.info("Organization %s" % org.name) org_setting = OrganizationSetting.objects.filter( organization=org)[0] manager = get_db_manager(org_setting.document_store) send_reminders_for_an_organization( org, on_date, sms_client, from_number=org_setting.sms_tel_number, dbm=manager) logger.info("Successfully sent reminders for Org: %s.", org.name) except Exception as e: logger.exception( "Error while sending reminders for organization : %s" % org.name) logger.info("Done sending all reminders.") except Exception as e: logger.exception("Exception while sending reminders")
def remove_deleted_ds_from_project(db_name): logger = logging.getLogger(db_name) try: dbm = get_db_manager(db_name) logger.info("starting data fix for " + db_name) all_data_senders = set(get_all_active_data_senders(dbm)) for project_doc in dbm.database.view("project_names/project_names", include_docs=True): try: project_data_senders = set(project_doc["doc"]["data_senders"]) invalid_ds = project_data_senders.difference(all_data_senders) project_doc = Project._wrap_row(project_doc) for ds in invalid_ds: logger.info("Found invalid data senders in project : " + str(project_doc) + " " + str(invalid_ds)) project_doc.delete_datasender(dbm, ds) except Exception as e: print "Error : " + db_name + " : " + str( project_doc) + e.message traceback.print_exc(file=sys.stdout) logger.info("done:" + db_name) mark_as_completed(db_name) except Exception as e: logger.exception("Failed Database : %s , with error :%s " % (db_name, e.message))
def handle(self, *args, **options): for database_name in document_stores_to_process(args): manager = get_db_manager(database_name) print ("Database %s") % (database_name) print "Syncing....." initializer.run(manager) print "Done."
def delete_questionnaires_without_projects(db_name): dbm = get_db_manager(db_name) logger = logging.getLogger(db_name) for row in dbm.database.query(list_all_formmodels_of_projects, include_docs=True): try: form_model_doc = row.doc if 'data_senders' in form_model_doc: logger.warn("Skipping migration - multiple runs") return projects = dbm.database.query(get_project_from_qid, include_docs=True, key=row.id) if not projects: submissions = dbm.database.query( get_submissions_for_formmodel, include_docs=True, key=form_model_doc['form_code']) for submission in submissions: logger.info("deleting submission with id:%s", submission.id) dbm.database.delete(submission.doc) logger.info("deleting form_model with id:%s and code:%s", row.id, form_model_doc['form_code']) dbm.database.delete(form_model_doc) except Exception as e: logger.error( 'something failed for for database : %s, project_doc with id: %s' % (dbm.database_name, row.id)) logger.error(e) mark_as_completed(db_name)
def convert_field_code_to_lower_case(db_name): logger = logging.getLogger(db_name) try: dbm = get_db_manager(db_name) rows = dbm.load_all_rows_in_view('questionnaire', reduce=False) for row in rows: form_model = FormModel.new_from_doc( dbm, FormModelDocument.wrap(row['value'])) is_upper = False for field in form_model.fields: if re.match(r".*[A-Z]+.*", field.code): logger.info("doc id: %s, field code: %s", form_model.id, field.code) is_upper = True field._dict['code'] = field.code.lower() if is_upper: form_model.save() survey_responses = survey_responses_by_form_code( dbm, form_model.form_code) for survey_response in survey_responses: convert_dict_keys_to_lowercase(survey_response.values) survey_response.save() logger.info("Modified survey response id: %s" % survey_response.uuid) except Exception as e: logger.exception(e.message) mark_as_completed(db_name)
def setUpClass(cls): cls.organization = DatabaseManager().get_organization_by_email(fetch_(from_(USERNAME), VALID_CREDENTIALS)) cls.organization.account_type = "Pro" cls.organization.save() HeadlessRunnerTest.setUpClass() cls.global_navigation = login(cls.driver) cls.dbm = get_db_manager('hni_testorg_slx364903')
def share_token(request, entity_type): manager = get_db_manager("public") organization_id = _get_organization_id(request) entity_preference = get_entity_preference(manager, organization_id, entity_type) if entity_preference: return HttpResponse(json.dumps({"token": entity_preference.share_token})) entity_preference = save_entity_preference(manager, organization_id, entity_type) return HttpResponse(json.dumps({"token": entity_preference.share_token}))
def migration_to_add_location_field_to_datasenders(db_name): logger = logging.getLogger(db_name) dbm = get_db_manager(db_name) mark_as_completed(db_name) logger.info('Migration started for database %s' % db_name) all_ds = get_all_entities(dbm, ['reporter']) for ds in all_ds: _add_location_field_if_absent(ds, dbm, logger)
def aggregate_test_questionnaire_submissions(db_name): logger = logging.getLogger(db_name) try: mark_as_completed(db_name) dbm = get_db_manager(db_name) aggregate_test_questionnaire(dbm) except Exception as e: logger.exception(e.message)
def test_should_create_organization_setting_with_document_store_on_create_organization( self): c = Client() reg_post = dict( email="*****@*****.**", first_name="first_name", last_name="last", organization_addressline1="x", organization_addressline2="y", organization_city="city", organization_country="country", organization_name="TEST_ORG_NAME", organization_office_phone="", organization_sector="PublicHealth", organization_state="state", organization_website="", organization_zipcode="zip", password1="a", password2="a", title="", ) response = c.post('/register/', reg_post) self.assertIsNotNone(response) user = User.objects.get(email='*****@*****.**') profile = RegistrationProfile.objects.get(user=user) activation_key = profile.activation_key response = c.post('/activate/%s/' % activation_key) self.assertIsNotNone(response) organization = Organization.objects.get(name="TEST_ORG_NAME") organization_settings = OrganizationSetting.objects.get( organization=organization) organization_db_name = organization_settings.document_store expected_org_db_name = ("hni_test_org_name_%s" % (organization.org_id, )).lower() self.assertEqual(expected_org_db_name, organization_db_name) couch_server = couchdb.client.Server(settings.COUCH_DB_SERVER) couch_server.resource.credentials = settings.COUCHDB_CREDENTIALS org_db = None try: org_db = couch_server[organization_db_name] except Exception: self.fail("Organization database %s not created" % (organization_db_name, )) self.assertIsNotNone(org_db) manager = get_db_manager(organization_db_name) test_reporter = reporters.find_reporter_entity( manager, TEST_REPORTER_MOBILE_NUMBER) self.assertIsNotNone(test_reporter) #clean up the org db del couch_server[organization_db_name]
def handle(self, *args, **options): for database_name in document_stores_to_process(args): print("Database %s") % (database_name, ) print "Syncing Views....." manager = get_db_manager(database_name) #force sync all views initializer.sync_views(manager) sync_views(manager) print "Done."
def migrate_form_model_to_add_eid_fields(db_name): logger = logging.getLogger(db_name) try: logger.info('Starting migration') dbm = get_db_manager(db_name) add_unique_id_and_short_code_field(dbm, logger) except Exception as e: logger.exception(e.message) mark_as_completed(db_name)
def create_org_database(db_name): from datawinners.initializer import run from datawinners.main.database import get_db_manager manager = get_db_manager(db_name) assert manager, "Could not create database manager for %s " % (db_name,) create_search_index(db_name) run(manager) return manager
def process_submission(database_name, rules, row): manager = get_db_manager(database_name) success = False submission = SurveyResponseDocument._wrap_row(row) for rule in rules: success = success or rule.update_submission(submission) if success: survey_response = SurveyResponse.new_from_doc(manager, submission) survey_response.save()
def migrate_to_merge_form_model_and_project(db_name): logger = logging.getLogger(db_name) try: logger.info('Starting migration') dbm = get_db_manager(db_name) merge_project_and_form_model_for(dbm, logger) except Exception as e: logger.exception(e.message) mark_as_completed(db_name)