def test_simple_submission_response(session: sa_orm.Session, extractor, simple_form: models.Form, simple_response_data: dict, user: models.User): expected_submission = factories.SubmissionFactory( form=simple_form, user=user, responses=simple_response_data) session.add(expected_submission) session.flush() actual_submissions = [s for s in extractor()] assert len(actual_submissions) == 1 actual_submission = actual_submissions[0] assert actual_submission == expected_submission
def test_make_processor(session: sa_orm.Session, processor_config, simple_form, simple_response_data): submission = factories.SubmissionFactory(form=simple_form, responses=simple_response_data) session.add(submission) session.flush() test_processor = factories.make_processor(session, processor_config) assert test_processor test_processor() session.flush() actual_num_events = session.query(models.ResponseEvent).count() assert actual_num_events == 1
def test_submission_timestamp(session, simple_form, user): now = utc_now() with freeze_time(now): submission = factories.SubmissionFactory(form=simple_form, user=user, responses={}) session.add(submission) session.flush() results = session.query(models.Submission).all() assert len(results) == 1 actual_submission = results[0] assert actual_submission.form == simple_form assert actual_submission.user == user assert actual_submission.date_created == now assert actual_submission.responses == {}
def test_json_transform_to_model(session, raw_data, transformer, mock_logger): timestamp_submission = utc_now() with freeze_time(timestamp_submission): form = factories.FormFactory(schema=raw_data.schema) submission = factories.SubmissionFactory(form=form, responses=raw_data.responses) session.add_all([form, submission]) session.flush() timestamp_transformation = utc_now() with freeze_time(timestamp_transformation): # TODO remove explicit processed_on to verify default results = transformer([submission], processed_on=timestamp_transformation) assert results assert isinstance(results, types.GeneratorType) results = list(results) assert len(results) == len(raw_data.events) # verify all standard fields for actual_event in results: assert isinstance(actual_event, models.ResponseEvent) assert actual_event.form_id == form.id assert actual_event.form_name == form.name assert actual_event.user_id == submission.user.id assert actual_event.user_full_name == submission.user.full_name assert actual_event.submission_id == submission.id assert actual_event.submission_created == timestamp_submission assert actual_event.processed_on == timestamp_transformation # convert results to a dictionary of only the data we care about sorted_actual_events = sorted(({ 'schema_path': e.schema_path, 'value': e.value, 'answer_type': e.answer_type.name, 'tag': None if not e.tag else e.tag } for e in results), key=lambda e: e['schema_path']) assert sorted_actual_events == raw_data.events _verify_logged_metrics(mock_logger, 1)
def test_json_transform_to_dict(session, raw_data, transformer): timestamp_submission = utc_now() with freeze_time(timestamp_submission): form = factories.FormFactory(schema=raw_data.schema) submission = factories.SubmissionFactory(form=form, responses=raw_data.responses) session.add_all([form, submission]) session.flush() timestamp_transformation = utc_now() with freeze_time(timestamp_transformation): # TODO remove explicit processed_on to verify default results = transformer([submission], processed_on=timestamp_transformation, to_dict=True) assert results assert isinstance(results, types.GeneratorType) results = list(results) assert len(results) == len(raw_data.events) # verify all standard fields for actual_event in results: assert isinstance(actual_event, dict) assert actual_event['form_id'] == form.id assert actual_event['form_name'] == form.name assert actual_event['user_id'] == submission.user.id assert actual_event['user_full_name'] == submission.user.full_name assert actual_event['submission_id'] == submission.id assert actual_event['submission_created'] == timestamp_submission assert actual_event['processed_on'] == timestamp_transformation sorted_actual_events = sorted(({ 'schema_path': e['schema_path'], 'value': e['value'], 'answer_type': e['answer_type'], 'tag': None if not e['tag'] else e['tag'] } for e in results), key=lambda e: e['schema_path']) assert sorted_actual_events == raw_data.events
def test_node_path_map_caching(monkeypatch, session, transformer, mock_logger, simple_form, num_responses_with_same_schema): # create responses for the basic form schema submissions = factories.SubmissionFactory.build_batch( num_responses_with_same_schema, form=simple_form, responses={}) for submission in submissions: submission.responses = {'basic_info': {'bmi': random.randint(10, 40)}} session.add_all(submissions) # create an unrelated response to a different form (uses a string response instead) adjusted_schema = copy.deepcopy(simple_form.schema) adjusted_schema['children'][0]['children'][0]['answerType'] = 'text' adjusted_schema['children'][0]['children'][0]['tag'] = 'member_bmi_as_text' unrelated_form = factories.FormFactory.build(name='unrelated', schema=adjusted_schema) session.add(unrelated_form) unreleated_submission = factories.SubmissionFactory( form=unrelated_form, responses={'basic_info': { 'bmi': 'my BMI is 40' }}) session.add(unreleated_submission) # create a response with an unrelated form that will not yield a cache hit empty_form = factories.FormFactory.build(schema={}) session.add(empty_form) empty_response = factories.SubmissionFactory.build(form=empty_form, responses={}) session.add(empty_response) all_submissions = session.query(models.Submission) # patch the cache handler so that we have direct access to the internal LRU cache for test assertions original_cache_func = transformers.get_node_path_map_cache generated_cache = None def _patched_make_cache_wrapper(session): nonlocal generated_cache generated_cache = original_cache_func(session) return generated_cache monkeypatch.setattr(transformers, 'get_node_path_map_cache', _patched_make_cache_wrapper) results = list(transformer(all_submissions)) # verify the logger recorded the correct number of responses processed _verify_logged_metrics(mock_logger, num_responses_with_same_schema + 2) # verify transformed events # # NOTES: only the unrelated form produces an additional event # the empty schema should not produce an event since there were no actual responses assert results assert len(results) == 1 + num_responses_with_same_schema same_schema_events = { e.submission_id: e for e in results if e.form_id == simple_form.id } for expected_submission in submissions: actual_event = same_schema_events.get(expected_submission.id) assert actual_event assert actual_event.schema_path == 'basic_info.bmi' assert int(actual_event.value ) == expected_submission.responses['basic_info']['bmi'] assert actual_event.tag == 'member_bmi' other_schema_events = { e.submission_id: e for e in results if e.form_id != simple_form.id } assert unreleated_submission.id in other_schema_events newer_submission_event = other_schema_events.get(unreleated_submission.id) assert newer_submission_event assert newer_submission_event.schema_path == 'basic_info.bmi' assert newer_submission_event.value == 'my BMI is 40' assert newer_submission_event.tag == 'member_bmi_as_text' # verify that the cache was used as expected assert generated_cache expected_cache_hits = max(0, num_responses_with_same_schema - 1) cache_info = generated_cache.cache_info() assert cache_info.hits == expected_cache_hits