def test_should_not_update_submissions_no_field_got_deleted(self):
dbm = Mock(spec=DatabaseManager)
form_code = 'code'
deleted_question_codes = []
with patch(
"datawinners.project.wizard_view.survey_responses_by_form_model_id"
) as survey_responses_by_form_model_id:
mock_document1 = SurveyResponseDocument(values={
'field_code': 'answer1',
'another_code': 'answer2'
})
mock_document2 = SurveyResponseDocument(values={
'field_code': 'answer3',
'another_code': 'answer4'
})
survey_responses_mock = [
(SurveyResponse.new_from_doc(dbm=None, doc=mock_document1)),
(SurveyResponse.new_from_doc(dbm=None, doc=mock_document2))
]
survey_responses_by_form_model_id.return_value = survey_responses_mock
remove_deleted_questions_from_submissions(dbm, form_code,
deleted_question_codes)
self.assertEquals(mock_document1.values, {
'field_code': 'answer1',
'another_code': 'answer2'
})
self.assertEquals(mock_document2.values, {
'field_code': 'answer3',
'another_code': 'answer4'
})
def test_should_update_the_associated_submission_when_question_code_is_updated(
self):
update_dict = {
"database_name": "database_name",
"old_form_code": "old_form_code",
"new_form_code": "new_form_code",
"new_revision": "new_revision"
}
with patch("datawinners.project.wizard_view.get_db_manager"
) as get_db_manager:
managerMock = Mock(spec=DatabaseManager)
get_db_manager.return_value = managerMock
managerMock._save_documents.return_value = []
with patch(
"datawinners.project.wizard_view.survey_responses_by_form_code"
) as survey_responses_by_form_code:
mock_document1 = SurveyResponseDocument()
mock_document2 = SurveyResponseDocument()
survey_responses_mock = [
(SurveyResponse.new_from_doc(dbm=None,
doc=mock_document1)),
(SurveyResponse.new_from_doc(dbm=None, doc=mock_document2))
]
survey_responses_by_form_code.return_value = survey_responses_mock
update_submissions_for_form_code_change(
managerMock, 'new_form_code', 'old_form_code')
managerMock._save_documents.assert_called_with(
[mock_document1, mock_document2])
self.assertEquals(mock_document1.form_code, "new_form_code")
self.assertEquals(mock_document2.form_code, "new_form_code")
def migrate(database):
try:
log_statement("database: %s" % database)
dbm = get_db_manager(SERVER, database=database)
revid_map = revsion_map(database, dbm)
for old_rev, values in revid_map.iteritems():
survey_response_docs = dbm.database.query(
map_survey_response_by_form_model_revision %
(values[1], old_rev))
for survey_response_doc in survey_response_docs:
survey_response = SurveyResponse.new_from_doc(
dbm=dbm,
doc=SurveyResponse.__document_class__.wrap(
survey_response_doc['value']))
log_statement("Changing revision on:%s from:%s to:%s" %
(survey_response.id,
survey_response.form_model_revision, values[0]))
survey_response.form_model_revision = values[0]
survey_response.save()
log_statement("Completed Database : %s" % database)
mark_as_completed(database)
log_file.writelines(
"\n=======================================================\n")
except Exception:
log_statement("Failed Database : %s" % database)
traceback.print_exc(file=log_file)
def get_survey_responses(dbm,
form_model_id,
from_time,
to_time,
page_number=0,
page_size=None,
view_name="surveyresponse"):
startkey, endkey = _get_start_and_end_key(form_model_id, from_time,
to_time)
if page_size is None:
rows = dbm.load_all_rows_in_view(view_name,
reduce=False,
descending=True,
startkey=startkey,
endkey=endkey)
else:
rows = dbm.load_all_rows_in_view(view_name,
reduce=False,
descending=True,
startkey=startkey,
endkey=endkey,
skip=page_number * page_size,
limit=page_size)
return [
SurveyResponse.new_from_doc(dbm=dbm,
doc=SurveyResponse.__document_class__.wrap(
row['value'])) for row in rows
]
def process_submission(database_name, rules, row):
manager = get_db_manager(database_name)
success = False
submission = SurveyResponseDocument._wrap_row(row)
for rule in rules:
success = success or rule.update_submission(submission)
if success:
survey_response = SurveyResponse.new_from_doc(manager, submission)
survey_response.save()
def migrate_db(self):
self.logger.info(
'\n ============================================= Start =============================================\n')
self.logger.info('Start db : %s' % self.dbm.database_name)
rows = self.dbm.database.iterview(UNDELETED_SURVEY_RESPONSE, BATCH_SIZE, reduce=False)
for row in rows:
survey_response_doc = SurveyResponse.__document_class__.wrap(row['value'])
survey_response = SurveyResponse.new_from_doc(dbm=self.dbm, doc=survey_response_doc)
self._create_feed_doc(survey_response)
self.logger.info(
'\n ========================================= End ==================================================\n')
def migrate_survey_response_with_form_code_as_delete(db_name):
logger = logging.getLogger(db_name)
logger.info('Starting Migration')
manager = get_db_manager(db_name)
for row in manager.database.query(survey_responses_with_delete_form_code):
try:
doc = SurveyResponseDocument.wrap(row['value'])
survey_response = SurveyResponse.new_from_doc(manager, doc)
logger.info("survey response id: %s" % survey_response.id)
survey_response.delete()
logger.info("Deleted survey response id: %s" % survey_response.id)
except Exception as e:
logger.exception("FAILED to delete:%s " % row['value']['_id'])
logger.info('Completed Migration')
def _process_survey_response(survey_response_doc, dbm, logger):
try:
survey_response = SurveyResponse.new_from_doc(
dbm=dbm,
doc=SurveyResponse.__document_class__.wrap(
survey_response_doc['value']))
if 'form_code' not in survey_response._doc:
logger.error("form_code not present in survey response:%s" %
survey_response.uuid)
return None
form_code = survey_response._doc['form_code']
form_models = _get_form_models(dbm, survey_response._doc['form_code'])
if not form_models:
logger.error(
"No Questionnaire found for survey response:%s with form_code: %s"
% survey_response.uuid, survey_response._doc['form_code'])
return None
elif len(form_models) > 1:
form_models.sort(key=lambda form_model: form_model._doc.created,
reverse=True)
matching_form_model = _get_matching_form_model(
survey_response.created, form_models)
if matching_form_model:
del survey_response._doc['form_code']
survey_response.form_model_id = matching_form_model.id
survey_response.save(process_post_update=False)
return form_code
else:
logger.error(
"No Questionnaire found with matching date for survey response: %s and form_code:%s"
% survey_response.uuid, survey_response._doc['form_code'])
return None
elif len(form_models) == 1:
del survey_response._doc['form_code']
survey_response.form_model_id = form_models[0].id
survey_response.save(process_post_update=False)
return form_code
except Exception as e:
logger.exception("Exception for survey response:%s" %
survey_response.uuid)
return None
def migrate_db(database):
mark_as_completed(database)
log_statement('\nStart migration on database : %s \n' % database)
try:
manager = get_db_manager(server=SERVER, database=database)
invalid_survey_response_docs = manager.database.query(map_invalid_survey_responses)
count = 0
log_statement('Total documents to be voided : %s' % len(invalid_survey_response_docs))
for survey_response_doc in invalid_survey_response_docs:
survey_response = SurveyResponse.new_from_doc(dbm=manager,
doc=SurveyResponse.__document_class__.wrap(survey_response_doc['value']))
survey_response.void()
count += 1
log_statement("Voided survey_response %s" % survey_response.uuid)
log_statement("Total number of documents voided are %s" % count)
log_statement('\nCompleted database : %s\n' % database)
except Exception as e:
log_statement('error:%s\n' % database)
traceback.print_exc(file=log_file)
def get_survey_responses_for_activity_period(dbm, form_model_id, from_time,
to_time):
from_time_in_epoch = convert_date_time_to_epoch(
from_time) if from_time is not None else None
to_time_in_epoch = convert_date_time_to_epoch(
to_time) if to_time is not None else None
startkey, endkey = _get_start_and_end_key(form_model_id,
from_time_in_epoch,
to_time_in_epoch)
rows = dbm.load_all_rows_in_view('survey_response_for_activity_period',
descending=True,
startkey=startkey,
endkey=endkey)
return [
SurveyResponse.new_from_doc(dbm=dbm,
doc=SurveyResponse.__document_class__.wrap(
row['value'])) for row in rows
]
def migrate_survey_response_to_add_owner(db_name):
logger = logging.getLogger(db_name)
try:
logger.info('Starting Migration')
mark_as_completed(db_name)
dbm = get_db_manager(db_name)
phone_to_rep_id_map, rep_id_to_uid_map = create_datasender_map(dbm)
org_id = OrganizationSetting.objects.get(document_store=dbm.database_name).organization_id
source_to_rep_id_map = add_email_info_to_datasender_map(phone_to_rep_id_map, org_id)
rows = dbm.database.view("surveyresponse/surveyresponse", reduce=False, include_docs=True)
for row in rows:
doc_id = row['value']['_id']
try:
original_source = row['value']['source']
except KeyError as e:
logger.info("Already migrated %s" % row['value']['_id']) #ignore, document already migrated
continue
doc = SurveyResponseDocument.wrap(row['value'])
survey_response = SurveyResponse.new_from_doc(dbm, doc)
data_sender_id = source_to_rep_id_map.get(original_source)
survey_response.created_by = data_sender_id
survey_response.modified_by = data_sender_id
owner_short_code = override_owner_with_on_behalf_user(rep_id_to_uid_map, data_sender_id, survey_response)
owner_uid = rep_id_to_uid_map.get(owner_short_code)
if owner_uid:
remove_attr_source_from_survey_response(survey_response)
else:
logger.warn("Unable to set owner_uid for source :" + original_source + " doc: " + doc_id)
survey_response.owner_uid = owner_uid
survey_response.save()
logger.info("Migrated %s" % survey_response.id)
except Exception as e:
logger.exception("Failed DB: %s with message %s" % (db_name, e.message))
logger.info('Completed Migration')
def _update_survey_response(dbm, row, reporter_id):
survey_response_doc = SurveyResponse.__document_class__.wrap(row['value'])
survey_response = SurveyResponse.new_from_doc(dbm=dbm, doc=survey_response_doc)
survey_response.is_anonymous_submission = False
survey_response.owner_uid = reporter_id
survey_response.save()
