class ModuleEngagementSummaryDataTaskMapTest(MapperTestMixin, TestCase): """Base class for test analysis of student engagement summaries""" task_class = ModuleEngagementSummaryDataTask input_record = ModuleEngagementRecord(course_id='foo/bar/baz', username='******', date=datetime.date(2015, 11, 1), entity_type='problem', entity_id='problem-id', event='attempted', count=1) def test_invalid_input_types(self): input_as_strings = list(self.input_record.to_string_tuple()) input_as_strings[-1] = 'foo' with self.assertRaises(ValueError): tuple(self.task.mapper('\t'.join(input_as_strings) + '\n')) def test_not_enough_input_columns(self): input_as_strings = list(self.input_record.to_string_tuple())[:-1] with self.assertRaises(ValueError): tuple(self.task.mapper('\t'.join(input_as_strings) + '\n')) def test_single_output(self): tsv_line = self.input_record.to_separated_values() self.assert_single_map_output(tsv_line, ('foo/bar/baz', 'foouser'), tsv_line)
class ModuleEngagementSummaryDataTaskReducerTest(ReducerTestMixin, TestCase): """Base class for test analysis of student engagement summaries""" task_class = ModuleEngagementSummaryDataTask output_record_type = ModuleEngagementSummaryRecord input_record = ModuleEngagementRecord(course_id='foo/bar/baz', username='******', date=datetime.date(2014, 03, 26), entity_type='problem', entity_id='problem-id', event='attempted', count=1) def setUp(self): super(ModuleEngagementSummaryDataTaskReducerTest, self).setUp() self.reduce_key = (self.COURSE_ID, 'test_user') self.video_play_record = self.input_record.replace( entity_type='video', entity_id='foox-video1', event='viewed', ) self.forum_record = self.input_record.replace(entity_type='discussion', entity_id='forum0', event='contributed') def test_output_format(self): self._check_output_complete_tuple( [self.input_record.to_separated_values()], (( 'foo/bar/baz', 'test_user', '2014-03-25', '2014-04-01', '1', '1', '0', 'inf', '0', '0', '1', ), )) def test_multiple_problem_attempts_same_problem(self): self._check_output_by_record_field( [ self.input_record.to_separated_values(), self.input_record.replace( date=datetime.date(2014, 03, 27)).to_separated_values() ], { 'problem_attempts': '2', 'problems_attempted': '1', 'problem_attempts_per_completed': 'inf', 'days_active': '2' }) def test_multiple_problem_attempts_different_problem(self): self._check_output_by_record_field( [ self.input_record.to_separated_values(), self.input_record.replace( entity_id='problem-id-2').to_separated_values() ], { 'problem_attempts': '2', 'problems_attempted': '2', 'problem_attempts_per_completed': 'inf', 'days_active': '1' }) def test_correct_problem_attempt(self): self._check_output_by_record_field( [ self.input_record.to_separated_values(), self.input_record.replace( event='completed').to_separated_values() ], { 'problem_attempts': '1', 'problems_attempted': '1', 'problems_completed': '1', 'problem_attempts_per_completed': str(1.0), 'days_active': '1' }) def test_correct_problem_multiple_attempts(self): self._check_output_by_record_field( [ self.input_record.replace(count=4).to_separated_values(), self.input_record.replace( event='completed').to_separated_values() ], { 'problem_attempts': '4', 'problems_attempted': '1', 'problems_completed': '1', 'problem_attempts_per_completed': str(4.0), 'days_active': '1' }) def test_multiple_correct_problems(self): self._check_output_by_record_field( [ self.input_record.replace(count=4).to_separated_values(), self.input_record.replace( event='completed').to_separated_values(), self.input_record.replace( date=datetime.date( 2014, 03, 27), entity_id='p2').to_separated_values(), self.input_record.replace( date=datetime.date(2014, 03, 27), entity_id='p2', event='completed', ).to_separated_values(), ], { 'problem_attempts': '5', 'problems_attempted': '2', 'problems_completed': '2', 'problem_attempts_per_completed': str(2.5), 'days_active': '2' }) def test_video_viewed(self): self._check_output_by_record_field( [ self.video_play_record.to_separated_values(), ], { 'problem_attempts': '0', 'problems_attempted': '0', 'problems_completed': '0', 'problem_attempts_per_completed': 'inf', 'videos_viewed': '1', 'days_active': '1' }) def test_multiple_videos_viewed(self): self._check_output_by_record_field([ self.video_play_record.to_separated_values(), self.video_play_record.replace( entity_id='1').to_separated_values(), ], {'videos_viewed': '2'}) def test_multiple_forum_contributions(self): self._check_output_by_record_field([ self.forum_record.to_separated_values(), self.forum_record.replace(entity_id='1', count=2).to_separated_values(), ], {'discussion_contributions': '3'})
def requires(self): kwargs = { 'warehouse_path': self.warehouse_path, 'overwrite': True, 'schema': self.schema, 'credentials': self.credentials, 'read_timeout': self.read_timeout, 'marker_schema': self.marker_schema, } yield ( LoadHiveTableToVertica( table_name='course_activity', sql_schema=CourseActivityRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='course_enrollment_daily', sql_schema=EnrollmentDailyRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='course_enrollment_birth_year_daily', sql_schema=EnrollmentByBirthYearRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='course_enrollment_education_level_daily', sql_schema=EnrollmentByEducationLevelRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='course_enrollment_gender_daily', sql_schema=EnrollmentByGenderRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='course_enrollment_mode_daily', sql_schema=EnrollmentByModeRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='course_meta_summary_enrollment', sql_schema=CourseSummaryEnrollmentRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='course_program_metadata', sql_schema=CourseProgramMetadataRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='course_enrollment_location_current', load_from_latest_partition=False, sql_schema=LastCountryPerCourseRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='module_engagement', sql_schema=ModuleEngagementRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='module_engagement_metric_ranges', sql_schema=ModuleEngagementSummaryMetricRangeRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='video_timeline', sql_schema=VideoTimelineRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='video', sql_schema=VideoSegmentSummaryRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='last_country_of_user_id', sql_schema=LastCountryOfUserRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='enterprise_enrollment', sql_schema=EnterpriseEnrollmentRecord.get_sql_schema(), **kwargs ), LoadHiveTableToVertica( table_name='enterprise_user', sql_schema=EnterpriseUserRecord.get_sql_schema(), **kwargs ) )
def requires(self): kwargs = { 'warehouse_path': self.warehouse_path, 'overwrite': True, 'schema': self.schema, 'credentials': self.credentials, 'read_timeout': self.read_timeout, 'marker_schema': self.marker_schema, } yield ( LoadHiveTableToVertica( table_name='course_activity', sql_schema=CourseActivityRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='course_enrollment_daily', sql_schema=EnrollmentDailyRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='course_enrollment_birth_year_daily', sql_schema=EnrollmentByBirthYearRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='course_enrollment_education_level_daily', sql_schema=EnrollmentByEducationLevelRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='course_enrollment_gender_daily', sql_schema=EnrollmentByGenderRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='course_enrollment_mode_daily', sql_schema=EnrollmentByModeRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='course_meta_summary_enrollment', sql_schema=CourseSummaryEnrollmentRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='course_program_metadata', sql_schema=CourseProgramMetadataRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='course_enrollment_location_current', load_from_latest_partition=False, sql_schema=LastCountryPerCourseRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='module_engagement', sql_schema=ModuleEngagementRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='module_engagement_metric_ranges', sql_schema=ModuleEngagementSummaryMetricRangeRecord. get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='video_timeline', sql_schema=VideoTimelineRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='video', sql_schema=VideoSegmentSummaryRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='last_country_of_user_id', sql_schema=LastCountryOfUserRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='enterprise_enrollment', sql_schema=EnterpriseEnrollmentRecord.get_sql_schema(), **kwargs), LoadHiveTableToVertica( table_name='enterprise_user', sql_schema=EnterpriseUserRecord.get_sql_schema(), **kwargs))