def make_feed_document_use_form_code_instead_of_form_model_id(db_name):
dbm = get_db_manager(db_name)
logger = logging.getLogger(db_name)
feed_dbm = get_feed_db_from_main_db_name(db_name)
questionnaires = dbm.load_all_rows_in_view('all_projects')
for questionnaire in questionnaires:
try:
feeds = feed_dbm.database.iterview(
"questionnaire_feed/questionnaire_feed",
1000,
startkey=[questionnaire.id],
endkey=[questionnaire.id, {}],
include_docs=True)
for feed in feeds:
try:
enriched_survey_response = EnrichedSurveyResponseDocument.wrap(
feed.doc)
enriched_survey_response.form_code = questionnaire[
'value']['form_code']
feed_dbm._save_document(
enriched_survey_response,
modified=enriched_survey_response.modified)
except Exception as e:
logger.exception("failed for feed:" + feed.id)
except Exception as e:
logger.exception("failed for questionnaire:" + questionnaire.id)
mark_as_completed(db_name)
def migrate_db(database):
log_statement('\nStart migration on database : %s \n' % database)
try:
manager = get_db_manager(server=SERVER,
database=database,
credentials=credentials)
questionnaire_form_model_docs = manager.database.query(
map_form_model_questionnaire)
mark_as_completed(database)
for form_model_doc in questionnaire_form_model_docs:
form_model = get_form_model(manager, form_model_doc)
log_statement(
"Process on :form_model document_id : %s , form code : %s" %
(form_model.id, form_model.form_code))
if any_field_is_optional_in(form_model):
migrate_form_model(form_model)
log_statement("Form Model updated :%s %s" %
(database, form_model.form_code))
log_statement(
"End process on :form_model document_id : %s , form code : %s"
% (form_model.id, form_model.form_code))
log_statement('\nEnd migration on database : %s\n' % database)
except Exception as e:
log_statement('error:%s:%s\n' % (e.message, database))
traceback.print_exc(file=log_file)
def convert_field_code_to_lower_case(db_name):
logger = logging.getLogger(db_name)
try:
dbm = get_db_manager(db_name)
rows = dbm.load_all_rows_in_view('questionnaire', reduce=False)
for row in rows:
form_model = FormModel.new_from_doc(
dbm, FormModelDocument.wrap(row['value']))
is_upper = False
for field in form_model.fields:
if re.match(r".*[A-Z]+.*", field.code):
logger.info("doc id: %s, field code: %s", form_model.id,
field.code)
is_upper = True
field._dict['code'] = field.code.lower()
if is_upper:
form_model.save()
survey_responses = survey_responses_by_form_code(
dbm, form_model.form_code)
for survey_response in survey_responses:
convert_dict_keys_to_lowercase(survey_response.values)
survey_response.save()
logger.info("Modified survey response id: %s" %
survey_response.uuid)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
def migration_to_add_email_data_for_web_users_in_couch(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
mark_as_completed(db_name)
manager = get_db_manager(db_name)
email_field_code = "email"
try:
form_model = get_form_model_by_code(manager, REGISTRATION_FORM_CODE)
except FormModelDoesNotExistsException as f:
logger.warning(f.message)
return
email_field_label = form_model._get_field_by_code(email_field_code).name
email_ddtype = form_model._get_field_by_code(email_field_code).ddtype
org_id = OrganizationSetting.objects.get(
document_store=manager.database_name).organization_id
user_profiles = NGOUserProfile.objects.filter(org_id=org_id)
for user_profile in user_profiles:
try:
short_code = user_profile.reporter_id
email_value = user_profile.user.email
data = (email_field_label, email_value, email_ddtype)
if short_code:
add_email_data_to_entity_document(manager, short_code, data,
logger)
except DataObjectAlreadyExists as e:
logger.warning(e.message)
except Exception as e:
logger.exception("FAILED to migrate: %s" % short_code)
def remove_deleted_ds_from_project(db_name):
logger = logging.getLogger(db_name)
try:
dbm = get_db_manager(db_name)
logger.info("starting data fix for " + db_name)
all_data_senders = set(get_all_active_data_senders(dbm))
for project_doc in dbm.database.view("project_names/project_names",
include_docs=True):
try:
project_data_senders = set(project_doc["doc"]["data_senders"])
invalid_ds = project_data_senders.difference(all_data_senders)
project_doc = Project._wrap_row(project_doc)
for ds in invalid_ds:
logger.info("Found invalid data senders in project : " +
str(project_doc) + " " + str(invalid_ds))
project_doc.delete_datasender(dbm, ds)
except Exception as e:
print "Error : " + db_name + " : " + str(
project_doc) + e.message
traceback.print_exc(file=sys.stdout)
logger.info("done:" + db_name)
mark_as_completed(db_name)
except Exception as e:
logger.exception("Failed Database : %s , with error :%s " %
(db_name, e.message))
def migrate(database):
try:
log_statement("database: %s" % database)
dbm = get_db_manager(SERVER, database=database)
revid_map = revsion_map(database, dbm)
for old_rev, values in revid_map.iteritems():
survey_response_docs = dbm.database.query(
map_survey_response_by_form_model_revision %
(values[1], old_rev))
for survey_response_doc in survey_response_docs:
survey_response = SurveyResponse.new_from_doc(
dbm=dbm,
doc=SurveyResponse.__document_class__.wrap(
survey_response_doc['value']))
log_statement("Changing revision on:%s from:%s to:%s" %
(survey_response.id,
survey_response.form_model_revision, values[0]))
survey_response.form_model_revision = values[0]
survey_response.save()
log_statement("Completed Database : %s" % database)
mark_as_completed(database)
log_file.writelines(
"\n=======================================================\n")
except Exception:
log_statement("Failed Database : %s" % database)
traceback.print_exc(file=log_file)
def delete_questionnaires_without_projects(db_name):
dbm = get_db_manager(db_name)
logger = logging.getLogger(db_name)
for row in dbm.database.query(list_all_formmodels_of_projects,
include_docs=True):
try:
form_model_doc = row.doc
if 'data_senders' in form_model_doc:
logger.warn("Skipping migration - multiple runs")
return
projects = dbm.database.query(get_project_from_qid,
include_docs=True,
key=row.id)
if not projects:
submissions = dbm.database.query(
get_submissions_for_formmodel,
include_docs=True,
key=form_model_doc['form_code'])
for submission in submissions:
logger.info("deleting submission with id:%s",
submission.id)
dbm.database.delete(submission.doc)
logger.info("deleting form_model with id:%s and code:%s",
row.id, form_model_doc['form_code'])
dbm.database.delete(form_model_doc)
except Exception as e:
logger.error(
'something failed for for database : %s, project_doc with id: %s'
% (dbm.database_name, row.id))
logger.error(e)
mark_as_completed(db_name)
def migration_to_add_email_data_for_web_users_in_couch(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
mark_as_completed(db_name)
manager = get_db_manager(db_name)
try:
form_model = get_form_model_by_code(manager, REGISTRATION_FORM_CODE)
except FormModelDoesNotExistsException as f:
logger.warning(f.message)
return
email_field = TextField(name=EMAIL_FIELD,
code=EMAIL_FIELD,
label="What is the subject's email",
defaultValue="",
instruction="Enter email id",
constraints=[TextLengthConstraint(max=50)],
required=False)
try:
form_model.add_field(email_field)
form_model.save()
logger.info("Migrated registration form")
except QuestionCodeAlreadyExistsException as e:
logger.warning('email field is present' + e.message)
except Exception as e:
logger.exception(e.message)
def migration_to_add_location_field_to_datasenders(db_name):
logger = logging.getLogger(db_name)
dbm = get_db_manager(db_name)
mark_as_completed(db_name)
logger.info('Migration started for database %s' % db_name)
all_ds = get_all_entities(dbm, ['reporter'])
for ds in all_ds:
_add_location_field_if_absent(ds, dbm, logger)
def migrate_survey_response_to_feed(db_name):
logger = logging.getLogger(db_name)
try:
logger.info("Starting migration")
mark_as_completed(db_name)
FeedBuilder(db_name, logger).migrate_db()
except Exception as e:
logger.exception("FAILED")
def aggregate_test_questionnaire_submissions(db_name):
logger = logging.getLogger(db_name)
try:
mark_as_completed(db_name)
dbm = get_db_manager(db_name)
aggregate_test_questionnaire(dbm)
except Exception as e:
logger.exception(e.message)
def migrate_form_model_to_add_eid_fields(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
add_unique_id_and_short_code_field(dbm, logger)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
def create_search_indices_for_submissions(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting indexing')
create_submission_index(db_name, logger)
logger.info('Completed Indexing')
mark_as_completed(db_name)
except Exception as e:
logger.exception(e.message)
def migration_to_update_constraints_for_datasender_registration(db_name):
logger = logging.getLogger(db_name)
dbm = get_db_manager(db_name)
try:
delete_and_create_form_model(dbm, GLOBAL_REGISTRATION_FORM_CODE)
mark_as_completed(db_name)
except Exception as e:
logger.exception("Datasender registration form re-creation failed for: %s with message %s" % (db_name, e.message))
logger.info('Completed Migration')
def migrate_to_merge_form_model_and_project(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
merge_project_and_form_model_for(dbm, logger)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
def update_counters_for_submissions(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
organization = OrganizationSetting.objects.get(
document_store=db_name).organization
update_sms_outgoing_charged_count_for_organization(organization)
mark_as_completed(db_name)
except Exception as e:
logger.exception(e.message)
def create_search_indices_for_subjects(db_name):
logger = logging.getLogger(db_name)
try:
mark_as_completed(db_name)
logger.info('Starting indexing')
recreate_index_for_db(db_name, es)
except Exception as e:
logger.exception("Failed DB: %s with message %s" %
(db_name, e.message))
logger.info('Completed Indexing')
def migrate_to_create_language_templates(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
delete_existing_templates(dbm)
create_language_template(dbm, logger)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
def create_search_indices_for_datasenders(db_name):
logger = logging.getLogger(db_name)
try:
mark_as_completed(db_name)
logger.info('Starting indexing')
create_datasender_index(db_name)
logger.info('Completed Indexing')
except FormModelDoesNotExistsException as e:
logger.warning(e.message)
except Exception as e:
logger.exception(e.message)
def seperate_datasender_and_contact_document(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
manager = get_db_manager(db_name)
for row in manager.database.query(datasender_document):
try:
row['value']['document_type'] = 'Contact'
manager.database.save(row['value'], process_post_update=False)
except Exception as e:
logger.error("Failed to update document with id:%s" % row['value']['_id'])
logger.info('Completed Migration')
mark_as_completed(db_name)
def migrate_db(db):
try:
mark_as_completed(db)
manager = get_db_manager(server=SERVER, database=db)
log_statement("Database: %s" % manager.database)
form_model = get_form_model_by_code(manager, ENTITY_DELETION_FORM_CODE)
log_statement("FormModel:%s" % form_model.id, manager.database)
for field in form_model.fields:
if field.code == 's':
field.set_constraints([])
form_model.save()
except Exception as e:
log_statement(":Error", db)
traceback.print_exc(file=log_file)
def make_survey_response_link_to_form_model_document_id(db_name):
dbm = get_db_manager(db_name)
logger = logging.getLogger(db_name)
form_codes = Set()
try:
for survey_response_doc in _get_survey_responses(dbm):
form_code = _process_survey_response(survey_response_doc, dbm,
logger)
form_codes.add(form_code)
_clear_form_models_from_cache(form_codes, dbm)
except Exception as e:
logger.exception(db_name)
mark_as_completed(db_name)
def update_mapping(database_name):
try:
dbm = get_db_manager(database_name)
logger = logging.getLogger(database_name)
for row in dbm.load_all_rows_in_view('questionnaire'):
form_model_doc = FormModelDocument.wrap(row["value"])
form_model_change_handler(form_model_doc, dbm)
except Exception as e:
logger.error(
"Mapping update failed for database %s for form model %s " %
(database_name, form_model_doc.form_code))
logger.error(e)
mark_as_completed(database_name)
def add_custom_group_field_to_data_sender_mapping(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
es = get_elasticsearch_handle()
fields = [
TextField(name="customgroups",
code='customgroups',
label='Custom groups')
]
es.put_mapping(db_name, 'reporter', get_fields_mapping('reg', fields))
logger.info('Completed Migration')
mark_as_completed(db_name)
def migrate_subject_form_code_to_add_regex_validator_in_short_code(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
mark_as_completed(db_name)
manager = get_db_manager(db_name)
for row in manager.database.query(map_subject_form_model):
try:
doc = FormModelDocument.wrap(row['value'])
form_model = FormModel.new_from_doc(manager, doc)
add_regex_constraint_to_short_code(form_model, logger)
except Exception as e:
logger.exception("FAILED to migrate:%s " % row['value']['_id'])
logger.info('Completed Migration')
def warm_up_contact_related_views(db_name):
logger = logging.getLogger(db_name)
try:
for view in views:
start_time = datetime.datetime.now()
response = _get_response_for_view(db_name, view)
logger.error('%s status:%d' % (view, response.status_code))
logger.error('%s:%s' % (view, response.text))
logger.error(
('%s time-taken:' + str(datetime.datetime.now() - start_time))
% view)
except Exception as e:
logger.exception(db_name)
mark_as_completed(db_name)
def find_deleted_questionnaires(db_name):
dbm = get_db_manager(db_name)
logger = logging.getLogger(db_name)
try:
questionnaires = dbm.load_all_rows_in_view('all_questionnaire')
for questionnaire in questionnaires:
if questionnaire['value']['void'] and questionnaire['value']['created'].year < 2014:
questionnaire_id = questionnaire['value']['_id']
rows = dbm.load_all_rows_in_view('surveyresponse', reduce=True, start_key=[questionnaire_id],
end_key=[questionnaire_id, {}])
if rows and len(rows) >= 1 and 'count' in rows[0]['value']:
logger.error("Deleted questionnaire %s Submission count : %d" % (questionnaire_id, rows[0]['value']['count']))
except Exception:
logger.exception()
mark_as_completed(db_name)
def delete_all_submission_logs(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
for row in dbm.database.query(list_all_submission_logs,
include_docs=True):
try:
dbm.database.delete(row.doc)
except Exception as e:
logging.error(
'Deletion failed for database : %s, doc with id: %s',
db_name, row['value']['_id'])
logging.error(e)
mark_as_completed(db_name)
except Exception as e:
logger.exception(e.message)
def undelete_datasenders_for_organization(db_name):
logger = logging.getLogger(db_name)
try:
dbm = get_db_manager(db_name)
for reporter in get_all_entities_include_voided(
dbm, [REPORTER_ENTITY_TYPE]):
try:
if reporter.is_void():
reporter._doc.void = False
reporter.save()
except Exception:
logger.exception("Save of reporter with id: %s failed" %
reporter.short_code)
except Exception:
logger.exception()
mark_as_completed(db_name)
def update_counters_for_submissions(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
year_month_submission_count_dict = get_submission_count_aggregate(dbm)
organization = OrganizationSetting.objects.get(document_store=dbm.database_name).organization
for key in year_month_submission_count_dict.keys():
year, month = int(key.split('_')[0]), int(key.split('_')[1])
date = datetime.date(year, month, 1)
update_counters_for_date(date, key, organization, year_month_submission_count_dict)
mark_as_completed(db_name)
logger.info("Completed")
except ResourceNotFound as e:
logger.warn(e.message)
except Exception as e:
logger.exception(e.message)