def render_map(request, share_token):
entity_preference = get_entity_preference_by_share_token(
get_db_manager("public"), share_token)
dbm = get_db_manager(entity_preference.org_id)
if not _flag_active("idnr_map",
entity_preference.org_id.split('_')[-1].upper()):
raise Http404
form_model = get_form_model_by_entity_type(
dbm, [entity_preference.entity_type.lower()])
return render_to_response('map.html', {
"entity_type":
entity_preference.entity_type,
"filters":
_get_filters(form_model, entity_preference.filters),
"geo_jsons":
geo_jsons(dbm, entity_preference.entity_type, request.GET,
entity_preference.details, entity_preference.specials),
"fallback_location":
entity_preference.fallback_location,
"mapbox_api_key":
get_mapbox_api_key(request.META['HTTP_HOST'])
},
context_instance=RequestContext(request))
def create_questionnaire_templates():
db_name = settings.QUESTIONNAIRE_TEMPLATE_DB_NAME
existing_dbm = get_db_manager(db_name)
_delete_db_and_remove_db_manager(existing_dbm)
recreated_dbm = get_db_manager(db_name)
_create_view(recreated_dbm)
file = settings.QUESTIONNAIRE_TEMPLATE_JSON_DATA_FILE
docs = create_template_from_json_file(recreated_dbm, file)
return docs
def handle(self, *args, **options):
get_cache_manager().flush_all()
for database_name in all_db_names():
print("Database %s") % (database_name, )
print 'Deleting...'
manager = get_db_manager(database_name)
_delete_db_and_remove_db_manager(manager)
for database_name in document_stores_to_process(args):
recreated_manager = get_db_manager(database_name)
print "Syncing Views....."
SyncOnlyChangedViews().sync_view(recreated_manager)
print "Loading data....."
load_data()
print "Done."
def recreate_index_for_db(database_name):
start = time.time()
es = get_elasticsearch_handle()
logger = logging.getLogger(database_name)
try:
es.delete_index(database_name)
except Exception as e:
logger.info("Could not delete index " + str(e.message))
response = es.create_index(database_name,
settings={
"number_of_shards": 1,
"number_of_replicas": 0
})
logger.info('%s search index created : %s' %
(database_name, response.get('ok')))
dbm = get_db_manager(database_name)
try:
create_all_mappings(dbm)
create_all_indices(dbm)
except Exception as e:
logger.exception("recreate index failed for database %s for" %
dbm.database_name)
logger.info(
'Time taken (seconds) for indexing {database_name} : {timetaken}'.
format(database_name=database_name, timetaken=(time.time() - start)))
def send_reminders_scheduled_on(on_date, sms_client):
"""
Sends out reminders scheduled for the given date, for each organization.
"""
assert isinstance(on_date, date)
try:
logger.info("Sending reminders for date:- %s" % on_date)
paid_organization = _get_active_paid_organizations()
for org in paid_organization:
try:
logger.info("Organization %s" % org.name)
org_setting = OrganizationSetting.objects.filter(
organization=org)[0]
smsc = org_setting.outgoing_number.smsc
charged_sms = smsc.vumi_username in settings.SMSC_WITHOUT_STATUS_REPORT
manager = get_db_manager(org_setting.document_store)
from_number = org_setting.sms_tel_number.split(',')[0]
send_reminders_for_an_organization(org,
on_date,
sms_client,
from_number=from_number,
dbm=manager,
charged_sms=charged_sms)
logger.info("Successfully sent reminders for Org: %s.",
org.name)
except Exception as e:
logger.exception(
"Error while sending reminders for organization : %s" %
org.name)
logger.info("Done sending all reminders.")
except Exception as e:
logger.exception("Exception while sending reminders")
def check_project_and_formmodel_name_mismatch(db_name):
try:
logging.info('Starting checking for database %s' % db_name)
dbm = get_db_manager(db_name)
check_for_name_mismatch(dbm)
except Exception as e:
logging.exception(e.message)
def update_associated_submissions(database_name, old_form_code, new_form_code, deleted_question_codes):
try:
manager = get_db_manager(database_name)
update_submissions_for_form_code_change(manager, new_form_code, old_form_code)
update_submissions_for_form_field_change(manager, old_form_code, deleted_question_codes)
except Exception as e:
current.retry(exc=e)
def create_search_indices_for_deleted_datasender(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting indexing')
dbm = get_db_manager(db_name)
es = get_elasticsearch_handle(timeout=600)
form_model = get_form_model_by_code(dbm, REGISTRATION_FORM_CODE)
datasenders = []
for entity in get_all_entities_include_voided(dbm,
REPORTER_ENTITY_TYPE):
if not entity.data:
continue
if entity.is_void() or entity.short_code == 'test':
datasender_dict = _create_datasender_dict(
dbm, entity, REPORTER, form_model)
datasender_dict.update({'id': entity.id})
datasenders.append(datasender_dict)
if datasenders:
es.bulk_index(dbm.database_name, REPORTER, datasenders)
logger.info('Created index for datasenders with ids :' +
str([a.get('id') for a in datasenders]))
logger.info('Completed Indexing')
mark_as_successful(db_name)
except Exception as e:
logger.exception(e.message)
def migration_to_add_email_data_for_web_users_in_couch(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
mark_as_completed(db_name)
manager = get_db_manager(db_name)
try:
form_model = get_form_model_by_code(manager, REGISTRATION_FORM_CODE)
except FormModelDoesNotExistsException as f:
logger.warning(f.message)
return
email_field = TextField(name=EMAIL_FIELD,
code=EMAIL_FIELD,
label="What is the subject's email",
defaultValue="",
instruction="Enter email id",
constraints=[TextLengthConstraint(max=50)],
required=False)
try:
form_model.add_field(email_field)
form_model.save()
logger.info("Migrated registration form")
except QuestionCodeAlreadyExistsException as e:
logger.warning('email field is present' + e.message)
except Exception as e:
logger.exception(e.message)
def migration_to_add_email_data_for_web_users_in_couch(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
mark_as_completed(db_name)
manager = get_db_manager(db_name)
email_field_code = "email"
try:
form_model = get_form_model_by_code(manager, REGISTRATION_FORM_CODE)
except FormModelDoesNotExistsException as f:
logger.warning(f.message)
return
email_field_label = form_model._get_field_by_code(email_field_code).name
email_ddtype = form_model._get_field_by_code(email_field_code).ddtype
org_id = OrganizationSetting.objects.get(
document_store=manager.database_name).organization_id
user_profiles = NGOUserProfile.objects.filter(org_id=org_id)
for user_profile in user_profiles:
try:
short_code = user_profile.reporter_id
email_value = user_profile.user.email
data = (email_field_label, email_value, email_ddtype)
if short_code:
add_email_data_to_entity_document(manager, short_code, data,
logger)
except DataObjectAlreadyExists as e:
logger.warning(e.message)
except Exception as e:
logger.exception("FAILED to migrate: %s" % short_code)
def send_reminders_scheduled_on(on_date, sms_client):
"""
Sends out reminders scheduled for the given date, for each organization.
"""
assert isinstance(on_date, date)
try:
logger.info("Sending reminders for date:- %s" % on_date)
paid_organization = _get_active_paid_organization()
for org in paid_organization:
try:
logger.info("Organization %s" % org.name)
org_setting = OrganizationSetting.objects.filter(
organization=org)[0]
manager = get_db_manager(org_setting.document_store)
send_reminders_for_an_organization(
org,
on_date,
sms_client,
from_number=org_setting.sms_tel_number,
dbm=manager)
logger.info("Successfully sent reminders for Org: %s.",
org.name)
except Exception as e:
logger.exception(
"Error while sending reminders for organization : %s" %
org.name)
logger.info("Done sending all reminders.")
except Exception as e:
logger.exception("Exception while sending reminders")
def remove_deleted_ds_from_project(db_name):
logger = logging.getLogger(db_name)
try:
dbm = get_db_manager(db_name)
logger.info("starting data fix for " + db_name)
all_data_senders = set(get_all_active_data_senders(dbm))
for project_doc in dbm.database.view("project_names/project_names",
include_docs=True):
try:
project_data_senders = set(project_doc["doc"]["data_senders"])
invalid_ds = project_data_senders.difference(all_data_senders)
project_doc = Project._wrap_row(project_doc)
for ds in invalid_ds:
logger.info("Found invalid data senders in project : " +
str(project_doc) + " " + str(invalid_ds))
project_doc.delete_datasender(dbm, ds)
except Exception as e:
print "Error : " + db_name + " : " + str(
project_doc) + e.message
traceback.print_exc(file=sys.stdout)
logger.info("done:" + db_name)
mark_as_completed(db_name)
except Exception as e:
logger.exception("Failed Database : %s , with error :%s " %
(db_name, e.message))
def handle(self, *args, **options):
for database_name in document_stores_to_process(args):
manager = get_db_manager(database_name)
print ("Database %s") % (database_name)
print "Syncing....."
initializer.run(manager)
print "Done."
def delete_questionnaires_without_projects(db_name):
dbm = get_db_manager(db_name)
logger = logging.getLogger(db_name)
for row in dbm.database.query(list_all_formmodels_of_projects,
include_docs=True):
try:
form_model_doc = row.doc
if 'data_senders' in form_model_doc:
logger.warn("Skipping migration - multiple runs")
return
projects = dbm.database.query(get_project_from_qid,
include_docs=True,
key=row.id)
if not projects:
submissions = dbm.database.query(
get_submissions_for_formmodel,
include_docs=True,
key=form_model_doc['form_code'])
for submission in submissions:
logger.info("deleting submission with id:%s",
submission.id)
dbm.database.delete(submission.doc)
logger.info("deleting form_model with id:%s and code:%s",
row.id, form_model_doc['form_code'])
dbm.database.delete(form_model_doc)
except Exception as e:
logger.error(
'something failed for for database : %s, project_doc with id: %s'
% (dbm.database_name, row.id))
logger.error(e)
mark_as_completed(db_name)
def convert_field_code_to_lower_case(db_name):
logger = logging.getLogger(db_name)
try:
dbm = get_db_manager(db_name)
rows = dbm.load_all_rows_in_view('questionnaire', reduce=False)
for row in rows:
form_model = FormModel.new_from_doc(
dbm, FormModelDocument.wrap(row['value']))
is_upper = False
for field in form_model.fields:
if re.match(r".*[A-Z]+.*", field.code):
logger.info("doc id: %s, field code: %s", form_model.id,
field.code)
is_upper = True
field._dict['code'] = field.code.lower()
if is_upper:
form_model.save()
survey_responses = survey_responses_by_form_code(
dbm, form_model.form_code)
for survey_response in survey_responses:
convert_dict_keys_to_lowercase(survey_response.values)
survey_response.save()
logger.info("Modified survey response id: %s" %
survey_response.uuid)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
def setUpClass(cls):
cls.organization = DatabaseManager().get_organization_by_email(fetch_(from_(USERNAME), VALID_CREDENTIALS))
cls.organization.account_type = "Pro"
cls.organization.save()
HeadlessRunnerTest.setUpClass()
cls.global_navigation = login(cls.driver)
cls.dbm = get_db_manager('hni_testorg_slx364903')
def share_token(request, entity_type):
manager = get_db_manager("public")
organization_id = _get_organization_id(request)
entity_preference = get_entity_preference(manager, organization_id, entity_type)
if entity_preference:
return HttpResponse(json.dumps({"token": entity_preference.share_token}))
entity_preference = save_entity_preference(manager, organization_id, entity_type)
return HttpResponse(json.dumps({"token": entity_preference.share_token}))
def migration_to_add_location_field_to_datasenders(db_name):
logger = logging.getLogger(db_name)
dbm = get_db_manager(db_name)
mark_as_completed(db_name)
logger.info('Migration started for database %s' % db_name)
all_ds = get_all_entities(dbm, ['reporter'])
for ds in all_ds:
_add_location_field_if_absent(ds, dbm, logger)
def aggregate_test_questionnaire_submissions(db_name):
logger = logging.getLogger(db_name)
try:
mark_as_completed(db_name)
dbm = get_db_manager(db_name)
aggregate_test_questionnaire(dbm)
except Exception as e:
logger.exception(e.message)
def test_should_create_organization_setting_with_document_store_on_create_organization(
self):
c = Client()
reg_post = dict(
email="*****@*****.**",
first_name="first_name",
last_name="last",
organization_addressline1="x",
organization_addressline2="y",
organization_city="city",
organization_country="country",
organization_name="TEST_ORG_NAME",
organization_office_phone="",
organization_sector="PublicHealth",
organization_state="state",
organization_website="",
organization_zipcode="zip",
password1="a",
password2="a",
title="",
)
response = c.post('/register/', reg_post)
self.assertIsNotNone(response)
user = User.objects.get(email='*****@*****.**')
profile = RegistrationProfile.objects.get(user=user)
activation_key = profile.activation_key
response = c.post('/activate/%s/' % activation_key)
self.assertIsNotNone(response)
organization = Organization.objects.get(name="TEST_ORG_NAME")
organization_settings = OrganizationSetting.objects.get(
organization=organization)
organization_db_name = organization_settings.document_store
expected_org_db_name = ("hni_test_org_name_%s" %
(organization.org_id, )).lower()
self.assertEqual(expected_org_db_name, organization_db_name)
couch_server = couchdb.client.Server(settings.COUCH_DB_SERVER)
couch_server.resource.credentials = settings.COUCHDB_CREDENTIALS
org_db = None
try:
org_db = couch_server[organization_db_name]
except Exception:
self.fail("Organization database %s not created" %
(organization_db_name, ))
self.assertIsNotNone(org_db)
manager = get_db_manager(organization_db_name)
test_reporter = reporters.find_reporter_entity(
manager, TEST_REPORTER_MOBILE_NUMBER)
self.assertIsNotNone(test_reporter)
#clean up the org db
del couch_server[organization_db_name]
def handle(self, *args, **options):
for database_name in document_stores_to_process(args):
print("Database %s") % (database_name, )
print "Syncing Views....."
manager = get_db_manager(database_name)
#force sync all views
initializer.sync_views(manager)
sync_views(manager)
print "Done."
def migrate_form_model_to_add_eid_fields(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
add_unique_id_and_short_code_field(dbm, logger)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
def create_org_database(db_name):
from datawinners.initializer import run
from datawinners.main.database import get_db_manager
manager = get_db_manager(db_name)
assert manager, "Could not create database manager for %s " % (db_name,)
create_search_index(db_name)
run(manager)
return manager
def process_submission(database_name, rules, row):
manager = get_db_manager(database_name)
success = False
submission = SurveyResponseDocument._wrap_row(row)
for rule in rules:
success = success or rule.update_submission(submission)
if success:
survey_response = SurveyResponse.new_from_doc(manager, submission)
survey_response.save()
def migrate_to_merge_form_model_and_project(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
merge_project_and_form_model_for(dbm, logger)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
