import logging
from mangrove.datastore.documents import FormModelDocument
from datawinners.main.couchdb.utils import all_db_names
from datawinners.search import form_model_change_handler
from datawinners.main.database import get_db_manager
from migration.couch.utils import migrate, mark_as_completed
def update_mapping(database_name):
try:
dbm = get_db_manager(database_name)
logger = logging.getLogger(database_name)
for row in dbm.load_all_rows_in_view('questionnaire'):
form_model_doc = FormModelDocument.wrap(row["value"])
form_model_change_handler(form_model_doc, dbm)
except Exception as e:
logger.error(
"Mapping update failed for database %s for form model %s " %
(database_name, form_model_doc.form_code))
logger.error(e)
mark_as_completed(database_name)
migrate(all_db_names(), update_mapping, version=(27, 2, 1), threads=2)
logger = logging.getLogger(db_name)
feed_dbm = get_feed_db_from_main_db_name(db_name)
questionnaires = dbm.load_all_rows_in_view('all_projects')
for questionnaire in questionnaires:
try:
feeds = feed_dbm.database.iterview(
"questionnaire_feed/questionnaire_feed",
1000,
startkey=[questionnaire.id],
endkey=[questionnaire.id, {}],
include_docs=True)
for feed in feeds:
try:
enriched_survey_response = EnrichedSurveyResponseDocument.wrap(
feed.doc)
enriched_survey_response.form_code = questionnaire[
'value']['form_code']
feed_dbm._save_document(
enriched_survey_response,
modified=enriched_survey_response.modified)
except Exception as e:
logger.exception("failed for feed:" + feed.id)
except Exception as e:
logger.exception("failed for questionnaire:" + questionnaire.id)
mark_as_completed(db_name)
migrate(all_db_names(),
make_feed_document_use_form_code_instead_of_form_model_id,
version=(17, 1, 1),
threads=3)
def create_language_template(dbm, logger):
create_custom_message_templates(dbm)
def delete_existing_templates(dbm):
customized_message_rows = dbm.load_all_rows_in_view('all_languages',
include_docs=True)
for row in customized_message_rows:
dbm.database.delete(row.doc)
account_message = dbm.database.get(ACCOUNT_MESSAGE_DOC_ID)
if account_message:
dbm.database.delete(account_message)
def migrate_to_create_language_templates(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
delete_existing_templates(dbm)
create_language_template(dbm, logger)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
migrate(all_db_names(),
migrate_to_create_language_templates,
version=(12, 0, 1),
threads=3)
def migration_to_add_email_data_for_web_users_in_couch(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
mark_as_completed(db_name)
manager = get_db_manager(db_name)
email_field_code = "email"
form_model = get_form_model_by_code(manager, "reg")
email_field_label = form_model._get_field_by_code(email_field_code).name
email_ddtype = form_model._get_field_by_code(email_field_code).ddtype
org_id = OrganizationSetting.objects.get(
document_store=manager.database_name).organization_id
user_profiles = NGOUserProfile.objects.filter(org_id=org_id)
for user_profile in user_profiles:
try:
short_code = user_profile.reporter_id
email_value = user_profile.user.email
data = (email_field_label, email_value, email_ddtype)
add_email_data_to_entity_document(manager, short_code, data)
except Exception as e:
logger.exception("FAILED to migrate:%s " % short_code)
migrate(all_db_names(),
migration_to_add_email_data_for_web_users_in_couch,
version=(9, 0, 1))
import sys
from datawinners.search.index_utils import get_elasticsearch_handle
if __name__ == "__main__" and __package__ is None:
sys.path.insert(0, ".")
from datawinners.main.couchdb.utils import all_db_names
from datawinners.main.management.commands.recreate_search_indexes import recreate_index_for_db
import logging
from migration.couch.utils import migrate, mark_as_completed
def create_search_indices_for_subjects(db_name):
logger = logging.getLogger(db_name)
try:
mark_as_completed(db_name)
logger.info('Starting indexing')
recreate_index_for_db(db_name, es)
except Exception as e:
logger.exception("Failed DB: %s with message %s" %
(db_name, e.message))
logger.info('Completed Indexing')
es = get_elasticsearch_handle()
migrate(all_db_names(),
create_search_indices_for_subjects,
version=(8, 0, 1),
threads=1)
logger.exception(e.message)
continue
if form_model.is_entity_registration_form(
) or "delete" == form_model.form_code:
continue
try:
es.delete_all(dbm.database_name, form_model.id)
except Exception as ignore:
logger.info("Error while deleting index %s" % ignore.message)
create_submission_mapping(dbm, form_model)
create_index(dbm, form_model, logger)
def create_search_indices_for_submissions(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting indexing')
create_submission_index(db_name, logger)
logger.info('Completed Indexing')
mark_as_completed(db_name)
except Exception as e:
logger.exception(e.message)
es = get_elasticsearch_handle(timeout=600)
migrate(all_db_names(),
create_search_indices_for_submissions,
version=(10, 0, 5),
threads=1)
survey_response_docs.append(search_dict)
if survey_response_docs:
es.bulk_index(dbm.database_name, form_model.id, survey_response_docs)
def create_index(database_name):
try:
dbm = get_db_manager(database_name)
logger = logging.getLogger(database_name)
for row in dbm.load_all_rows_in_view('questionnaire'):
form_model_doc = FormModelDocument.wrap(row["value"])
form_model_change_handler(form_model_doc, dbm)
try:
create_submission_index(dbm, row)
except Exception as e:
logger.error(
"Index update failed for database %s and for formmodel %s"
% (database_name, row.id))
logger.error(e)
except Exception as e:
logger.error(
"Mapping update failed for database %s for form model %s " %
(database_name, form_model_doc.form_code))
logger.error(e)
mark_as_completed(database_name)
migrate(all_db_names(), create_index, version=(11, 0, 6), threads=2)
import sys
from datawinners.main.couchdb.utils import all_db_names
if __name__ == "__main__" and __package__ is None:
sys.path.insert(0, ".")
import logging
from datawinners.feeds.migrate import FeedBuilder
from migration.couch.utils import mark_as_completed, migrate
def migrate_survey_response_to_feed(db_name):
logger = logging.getLogger(db_name)
try:
logger.info("Starting migration")
mark_as_completed(db_name)
FeedBuilder(db_name, logger).migrate_db()
except Exception as e:
logger.exception("FAILED")
migrate(all_db_names(), migrate_survey_response_to_feed, version=(7, 0, 3))
try:
del form_model_doc['state']
except KeyError as e:
logger.warn(e)
dbm._save_document(form_model_doc)
update_reminders(dbm, project_data, logger)
logger.info("Deleting project with id: %s", row.id)
dbm.database.delete(row.doc)
except Exception as e:
logger.error(
'Merging project and form_model failed for database : %s, project_doc with id: %s',
dbm.database_name, row.id)
logger.error(e)
def migrate_to_merge_form_model_and_project(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
merge_project_and_form_model_for(dbm, logger)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
migrate(all_db_names(),
migrate_to_merge_form_model_and_project,
version=(11, 0, 4),
threads=3)
from mangrove.transport.contract.survey_response import SurveyResponse
survey_responses_with_delete_form_code = """
function(doc) {
if (doc.document_type == "SurveyResponse" && doc.form_code == 'delete') {
emit(doc._id, doc);
}
}"""
def migrate_survey_response_with_form_code_as_delete(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
manager = get_db_manager(db_name)
for row in manager.database.query(survey_responses_with_delete_form_code):
try:
doc = SurveyResponseDocument.wrap(row['value'])
survey_response = SurveyResponse.new_from_doc(manager, doc)
logger.info("survey response id: %s" % survey_response.id)
survey_response.delete()
logger.info("Deleted survey response id: %s" % survey_response.id)
except Exception as e:
logger.exception("FAILED to delete:%s " % row['value']['_id'])
logger.info('Completed Migration')
migrate(all_db_names(),
migrate_survey_response_with_form_code_as_delete,
version=(7, 0, 1))
from datawinners.main.database import get_db_manager
from migration.couch.utils import migrate, mark_as_completed
def advanced_questionnaire_usage(db_name):
dbm = get_db_manager(db_name)
logger = logging.getLogger(db_name)
try:
questionnaires = dbm.load_all_rows_in_view('all_projects')
for questionnaire in questionnaires:
if 'xform' in questionnaire['value'] and questionnaire['value'][
'xform']:
questionnaire_id = questionnaire['value']['_id']
rows = dbm.load_all_rows_in_view(
'surveyresponse',
reduce=True,
start_key=[questionnaire_id],
end_key=[questionnaire_id, {}])
if rows and len(rows) >= 1 and 'count' in rows[0]['value']:
logger.error("Questionnaire: %s Submission count : %d" %
(questionnaire_id, rows[0]['value']['count']))
except Exception:
logger.exception()
mark_as_completed(db_name)
migrate(all_db_names(),
advanced_questionnaire_usage,
version=(14, 0, 2),
threads=6)
from datawinners.main.database import get_db_manager
if __name__ == "__main__" and __package__ is None:
sys.path.insert(0, ".")
from datawinners.main.couchdb.utils import all_db_names
import logging
from migration.couch.utils import migrate, mark_start_of_migration
es = get_elasticsearch_handle()
def delete_submission_mapping(db_name):
logger = logging.getLogger(db_name)
try:
mark_start_of_migration(db_name)
logger.info('Starting migration')
dbm = get_db_manager(db_name)
rows = dbm.load_all_rows_in_view("questionnaire")
for row in rows:
index_name = dbm.database_name
doc_type = row['value']['_id']
try:
es.send_request('DELETE', [index_name, doc_type, '_mapping'])
except Exception as e:
logging.error('Failed to delete mapping for index: %s and doctype: %s', index_name, doc_type)
logging.error("exception %s", e)
pass
except Exception as e:
logger.exception(e.message)
migrate(all_db_names(), delete_submission_mapping, version=(10, 0, 8), threads=1)
logger.info("Deleting user with id: %s", profile_reporter_id)
organization = Organization.objects.get(org_id=user_profile.org_id)
if organization.in_trial_mode:
logger.info("Deleting trail user with id: %s", profile_reporter_id)
# delete_datasender_for_trial_mode(dbm, [profile_reporter_id.lower()], 'reporter')
def delete_voided_datasenders(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
for row in dbm.database.query(map_voided_datasenders):
try:
user_profiles = NGOUserProfile.objects.filter(
reporter_id=row['key'])
user = user_profiles[0].user
if _is_only_datasender(user):
_delete_user_entry(dbm, user_profiles[0], logger)
except IndexError:
logger.info("User with reporter-id: '%s' does not exist" %
row['key'])
except Exception as e:
logger.exception(e.message)
migrate(all_db_names(),
delete_voided_datasenders,
version=(10, 1, 4),
threads=1)
f.pop('event_time_field_flag')
#Remove entity type from questionnaire form models.
if document_data.get('entity_type'):
document_data.pop('entity_type')
if short_code_dict:
json_data.remove(short_code_dict)
form_model = FormModel.new_from_doc(dbm, (FormModelDocument.wrap(document_data)))
if short_code_field:
form_model._form_fields.insert(index, short_code_field)
if validator:
form_model.add_validator(validator)
_save_form_model_doc(dbm, form_model)
except Exception as e:
logger.error('Failed form model for database : %s, doc with id: %s', dbm.database_name,
row.id)
logger.error(e)
def migrate_form_model_to_add_eid_fields(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
dbm = get_db_manager(db_name)
add_unique_id_and_short_code_field(dbm, logger)
except Exception as e:
logger.exception(e.message)
mark_as_completed(db_name)
migrate(all_db_names(), migrate_form_model_to_add_eid_fields, version=(11, 0, 3), threads=3)
]
for rep_id in rep_ids_for_migration:
reporter_entity = contact_by_short_code(dbm, rep_id)
_void_existing_data_records(dbm, rep_id)
reporter_entity.add_data(
data=_create_data_from_entity(reporter_entity),
submission={"form_code": "reg"})
mark_as_completed(database_name)
logger.info(
'Time taken (seconds) for migrating {database_name} : {timetaken}'.
format(database_name=database_name,
timetaken=(time.time() - start)))
except Exception as e:
logger.exception(
'Unexpected error while migrating data records for users',
e.message)
raise
logger.info('Started data record migration for user records')
start = time.time()
migrate(["hni_testorg_slx364903"],
recreate_data_records_for_account_users,
version=(32, 1, 0),
threads=1)
logger.info('Completed data record migration for user records')
logger.info('Total Time taken (seconds) : {timetaken}'.format(
timetaken=(time.time() - start)))
import logging
from migration.couch.utils import migrate, mark_as_completed
def _add_location_field_if_absent(datasender, dbm, logger):
if "location" not in datasender.data and '_geo' in datasender.aggregation_paths.keys(
):
logger.info('Adding location field to datasender %s' %
datasender.short_code)
data = ("location", datasender.aggregation_paths["_geo"],
get_datadict_type_by_slug(dbm, slug='location'))
datasender.update_latest_data([data])
else:
logger.info('Location field already present for datasender %s' %
datasender.short_code)
def migration_to_add_location_field_to_datasenders(db_name):
logger = logging.getLogger(db_name)
dbm = get_db_manager(db_name)
mark_as_completed(db_name)
logger.info('Migration started for database %s' % db_name)
all_ds = get_all_entities(dbm, ['reporter'])
for ds in all_ds:
_add_location_field_if_absent(ds, dbm, logger)
migrate(all_db_names(),
migration_to_add_location_field_to_datasenders,
version=(9, 0, 6))
import logging
from mangrove.contrib.registration import GLOBAL_REGISTRATION_FORM_CODE
from mangrove.utils.test_utils.database_utils import delete_and_create_form_model
from datawinners.main.database import get_db_manager
from datawinners.main.couchdb.utils import all_db_names
from migration.couch.utils import migrate, mark_as_completed
def migration_to_update_constraints_for_datasender_registration(db_name):
logger = logging.getLogger(db_name)
dbm = get_db_manager(db_name)
try:
delete_and_create_form_model(dbm, GLOBAL_REGISTRATION_FORM_CODE)
mark_as_completed(db_name)
except Exception as e:
logger.exception(
"Datasender registration form re-creation failed for: %s with message %s"
% (db_name, e.message))
logger.info('Completed Migration')
migrate(all_db_names(),
migration_to_update_constraints_for_datasender_registration,
version=(22, 0, 2),
threads=3)
import logging
from datawinners.accountmanagement.models import OrganizationSetting
from datawinners.main.couchdb.utils import all_db_names
from migration.couch.utils import migrate, mark_as_completed
def update_sms_outgoing_charged_count_for_organization(organization):
message_trackers = organization._get_all_message_trackers()
for message_tracker in message_trackers:
message_tracker.outgoing_sms_charged_count = message_tracker.outgoing_sms_count
message_tracker.save()
def update_counters_for_submissions(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting migration')
organization = OrganizationSetting.objects.get(
document_store=db_name).organization
update_sms_outgoing_charged_count_for_organization(organization)
mark_as_completed(db_name)
except Exception as e:
logger.exception(e.message)
migrate(all_db_names(),
update_counters_for_submissions,
version=(10, 0, 7),
threads=1)
logger.info('Starting indexing')
dbm = get_db_manager(db_name)
es = get_elasticsearch_handle(timeout=600)
form_model = get_form_model_by_code(dbm, REGISTRATION_FORM_CODE)
datasenders = []
for entity in get_all_entities_include_voided(dbm,
REPORTER_ENTITY_TYPE):
if not entity.data:
continue
if entity.is_void() or entity.short_code == 'test':
datasender_dict = _create_datasender_dict(
dbm, entity, REPORTER, form_model)
datasender_dict.update({'id': entity.id})
datasenders.append(datasender_dict)
if datasenders:
es.bulk_index(dbm.database_name, REPORTER, datasenders)
logger.info('Created index for datasenders with ids :' +
str([a.get('id') for a in datasenders]))
logger.info('Completed Indexing')
mark_as_successful(db_name)
except Exception as e:
logger.exception(e.message)
es = get_elasticsearch_handle(timeout=600)
migrate(all_db_names(),
create_search_indices_for_deleted_datasender,
version=(10, 0, 2),
threads=1)
]
def _get_response_for_view(db_name, view_name):
return requests.get(settings.COUCH_DB_SERVER + "/" +
(View_pah % (db_name, view_name, view_name)),
auth=settings.COUCHDBMAIN_CREDENTIALS,
timeout=TIMEOUT_IN_SECONDS)
def warm_up_contact_related_views(db_name):
logger = logging.getLogger(db_name)
try:
for view in views:
start_time = datetime.datetime.now()
response = _get_response_for_view(db_name, view)
logger.error('%s status:%d' % (view, response.status_code))
logger.error('%s:%s' % (view, response.text))
logger.error(
('%s time-taken:' + str(datetime.datetime.now() - start_time))
% view)
except Exception as e:
logger.exception(db_name)
mark_as_completed(db_name)
migrate(all_db_names(),
warm_up_contact_related_views,
version=(22, 0, 4),
threads=5)
}"""
def add_regex_constraint_to_short_code(form_model, logger):
form_model.entity_question.set_constraints([
TextLengthConstraint(max=20)._to_json(),
ShortCodeRegexConstraint("^[a-zA-Z0-9]+$")._to_json()
])
form_model.save()
logger.info("migrated form code: %s" % form_model.form_code)
def migrate_subject_form_code_to_add_regex_validator_in_short_code(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
mark_as_completed(db_name)
manager = get_db_manager(db_name)
for row in manager.database.query(map_subject_form_model):
try:
doc = FormModelDocument.wrap(row['value'])
form_model = FormModel.new_from_doc(manager, doc)
add_regex_constraint_to_short_code(form_model, logger)
except Exception as e:
logger.exception("FAILED to migrate:%s " % row['value']['_id'])
logger.info('Completed Migration')
migrate(all_db_names(),
migrate_subject_form_code_to_add_regex_validator_in_short_code,
version=(8, 1, 0))
def _clear_form_models_from_cache(form_codes, dbm):
cache_manger = get_cache_manager()
keys = [
get_form_model_cache_key(form_code, dbm) for form_code in form_codes
if form_code
]
if keys:
cache_manger.delete_multi(keys)
def make_survey_response_link_to_form_model_document_id(db_name):
dbm = get_db_manager(db_name)
logger = logging.getLogger(db_name)
form_codes = Set()
try:
for survey_response_doc in _get_survey_responses(dbm):
form_code = _process_survey_response(survey_response_doc, dbm,
logger)
form_codes.add(form_code)
_clear_form_models_from_cache(form_codes, dbm)
except Exception as e:
logger.exception(db_name)
mark_as_completed(db_name)
migrate(all_db_names(),
make_survey_response_link_to_form_model_document_id,
version=(13, 1, 2),
threads=3)
import sys
from mangrove.contrib.registration import GLOBAL_REGISTRATION_FORM_CODE
from mangrove.utils.test_utils.database_utils import delete_and_create_form_model
if __name__ == "__main__" and __package__ is None:
sys.path.insert(0, ".")
from datawinners.main.database import get_db_manager
from datawinners.main.couchdb.utils import all_db_names
import logging
from migration.couch.utils import migrate, mark_as_completed
def migration_to_update_constraints_for_datasender_registration(db_name):
logger = logging.getLogger(db_name)
dbm = get_db_manager(db_name)
try:
mark_as_completed(db_name)
delete_and_create_form_model(dbm, GLOBAL_REGISTRATION_FORM_CODE)
except Exception as e:
logger.exception(
"Failed datasender registration form creation for: %s with message %s"
% (db_name, e.message))
logger.info('Completed Migration')
migrate(all_db_names(),
migration_to_update_constraints_for_datasender_registration,
version=(9, 0, 5))
organization__in=org_ids_with_reminders).values_list('document_store')
]
datastore_org_id_map = dict(zip(organization_names, org_ids_with_reminders))
def remove_reminders_for_deleted_questionnaires(db_name):
logger = logging.getLogger(db_name)
try:
dbm = get_db_manager(db_name)
org_id = datastore_org_id_map[db_name]
reminders = Reminder.objects.filter(organization=org_id)
for reminder in reminders:
try:
questionnaire = Project.get(dbm, reminder.project_id)
if not questionnaire._doc['void'] and questionnaire._doc[
'reminder_and_deadline'][
'deadline_type'] == 'Following':
logger.error(
"Questionnaire Id %s for reminder %s has following setup"
% (reminder.project_id, reminder.id))
except DataObjectNotFound:
pass
except Exception:
logger.exception(db_name)
mark_as_completed(db_name)
migrate(organization_names,
remove_reminders_for_deleted_questionnaires,
version=(14, 0, 4),
threads=1)
from migration.couch.utils import migrate, mark_as_completed
datasender_document = """
function(doc) {
if (doc.document_type == "Entity" && doc.aggregation_paths['_type'] == "reporter") {
emit(doc.short_code, doc);
}
}"""
def add_custom_group_field_to_data_sender_mapping(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
es = get_elasticsearch_handle()
fields = [
TextField(name="customgroups",
code='customgroups',
label='Custom groups')
]
es.put_mapping(db_name, 'reporter', get_fields_mapping('reg', fields))
logger.info('Completed Migration')
mark_as_completed(db_name)
migrate(all_db_names(),
add_custom_group_field_to_data_sender_mapping,
version=(22, 0, 3),
threads=3)
logger = logging.getLogger(db_name)
try:
dbm = get_db_manager(db_name)
logger.info("starting data fix for " + db_name)
all_data_senders = set(get_all_active_data_senders(dbm))
for project_doc in dbm.database.view("project_names/project_names",
include_docs=True):
try:
project_data_senders = set(project_doc["doc"]["data_senders"])
invalid_ds = project_data_senders.difference(all_data_senders)
project_doc = Project._wrap_row(project_doc)
for ds in invalid_ds:
logger.info("Found invalid data senders in project : " +
str(project_doc) + " " + str(invalid_ds))
project_doc.delete_datasender(dbm, ds)
except Exception as e:
print "Error : " + db_name + " : " + str(
project_doc) + e.message
traceback.print_exc(file=sys.stdout)
logger.info("done:" + db_name)
mark_as_completed(db_name)
except Exception as e:
logger.exception("Failed Database : %s , with error :%s " %
(db_name, e.message))
migrate(all_db_names(), remove_deleted_ds_from_project, version=(7, 0, 4))
processed = []
for form_model_doc in subject_form_model_docs:
form_model = get_instance_from_doc(manager,
form_model_doc['value'])
if form_model.form_code in processed:
continue
processed.append(form_model.form_code)
logger.info("Process on :form_model: %s, form code : %s" %
(form_model.id, form_model.form_code))
data_to_restore, current_data_length = renumber_fields_name(
form_model)
datarecord_docs = manager.database.query(
map_datarecord_by_form_code,
key=[form_model.form_code, current_data_length])
for datarecord_doc in datarecord_docs:
migrate_entity(manager, form_model, datarecord_doc,
data_to_restore)
form_model.save()
logger.info("End process on :form_model: %s , form code : %s" %
(form_model.id, form_model.form_code))
logger.info('End migration on database')
except Exception as e:
logger.exception(e.message)
migrate(all_db_names(), migrate_story_2099, version=(7, 0, 5))
from mangrove.errors.MangroveException import FormModelDoesNotExistsException
if __name__ == "__main__" and __package__ is None:
sys.path.insert(0, ".")
from datawinners.main.couchdb.utils import all_db_names
from datawinners.search.datasender_index import create_datasender_index
import logging
from migration.couch.utils import migrate, mark_as_completed
def create_search_indices_for_datasenders(db_name):
logger = logging.getLogger(db_name)
try:
mark_as_completed(db_name)
logger.info('Starting indexing')
create_datasender_index(db_name)
logger.info('Completed Indexing')
except FormModelDoesNotExistsException as e:
logger.warning(e.message)
except Exception as e:
logger.exception(e.message)
es = get_elasticsearch_handle()
migrate(all_db_names(),
create_search_indices_for_datasenders,
version=(9, 0, 7),
threads=1)
logger = logging.getLogger(db_name)
feed_dbm = get_feed_db_from_main_db_name(db_name)
questionnaires = dbm.load_all_rows_in_view('all_projects')
for questionnaire in questionnaires:
try:
feeds = feed_dbm.database.iterview(
"questionnaire_feed/questionnaire_feed",
1000,
startkey=[questionnaire.id],
endkey=[questionnaire.id, {}],
include_docs=True)
for feed in feeds:
try:
enriched_survey_response = EnrichedSurveyResponseDocument.wrap(
feed.doc)
enriched_survey_response.form_code = questionnaire[
'value']['form_code']
feed_dbm._save_document(
enriched_survey_response,
modified=enriched_survey_response.modified)
except Exception as e:
logger.exception("failed for feed:" + feed.id)
except Exception as e:
logger.exception("failed for questionnaire:" + questionnaire.id)
mark_as_completed(db_name)
migrate(db_names,
make_feed_document_use_form_code_instead_of_form_model_id,
version=(13, 1, 3),
threads=3)
if 'data_senders' in form_model_doc:
logger.warn("Skipping migration - multiple runs")
return
projects = dbm.database.query(get_project_from_qid,
include_docs=True,
key=row.id)
if not projects:
submissions = dbm.database.query(
get_submissions_for_formmodel,
include_docs=True,
key=form_model_doc['form_code'])
for submission in submissions:
logger.info("deleting submission with id:%s",
submission.id)
dbm.database.delete(submission.doc)
logger.info("deleting form_model with id:%s and code:%s",
row.id, form_model_doc['form_code'])
dbm.database.delete(form_model_doc)
except Exception as e:
logger.error(
'something failed for for database : %s, project_doc with id: %s'
% (dbm.database_name, row.id))
logger.error(e)
mark_as_completed(db_name)
migrate(all_db_names(),
delete_questionnaires_without_projects,
version=(11, 0, 2),
threads=3)