def create_validation_task(self,
generate_before=True,
tuple_output=True,
include_nonstate_changes=True,
earliest_timestamp=None,
expected_validation=None):
"""Create a task for testing purposes."""
interval = '2013-01-01-2014-10-10'
interval_value = luigi.DateIntervalParameter().parse(interval)
earliest_timestamp_value = luigi.DateHourParameter().parse(
earliest_timestamp) if earliest_timestamp else None
expected_validation_value = (
luigi.DateHourParameter().parse(expected_validation)
if expected_validation else None)
self.task = CourseEnrollmentValidationTask(
interval=interval_value,
output_root="/fake/output",
generate_before=generate_before,
tuple_output=tuple_output,
include_nonstate_changes=include_nonstate_changes,
earliest_timestamp=earliest_timestamp_value,
expected_validation=expected_validation_value,
)
self.task.init_local()
class BaseCourseEnrollmentValidationTaskReducerTest(ReducerTestMixin, unittest.TestCase):
"""Provide common methods for testing CourseEnrollmentValidationTask reducer."""
def setUp(self):
self.task_class = CourseEnrollmentValidationTask
super(BaseCourseEnrollmentValidationTaskReducerTest, self).setUp()
self.reduce_key = ('foo/bar/baz', 0)
self.mode = 'honor'
def create_task(self, generate_before=True, tuple_output=True, include_nonstate_changes=True,
earliest_timestamp=None, expected_validation=None):
"""Create a task for testing purposes."""
interval = '2013-01-01-2014-10-10'
interval_value = luigi.DateIntervalParameter().parse(interval)
earliest_timestamp_value = luigi.DateHourParameter().parse(earliest_timestamp) if earliest_timestamp else None
expected_validation_value = (
luigi.DateHourParameter().parse(expected_validation) if expected_validation else None
)
self.task = CourseEnrollmentValidationTask(
interval=interval_value,
output_root="/fake/output",
generate_before=generate_before,
tuple_output=tuple_output,
include_nonstate_changes=include_nonstate_changes,
earliest_timestamp=earliest_timestamp_value,
expected_validation=expected_validation_value,
)
self.task.init_local()
def _activated(self, timestamp, mode=None):
"""Creates an ACTIVATED event."""
return (timestamp, ACTIVATED, mode or self.mode, None)
def _deactivated(self, timestamp, mode=None):
"""Creates a DEACTIVATED event."""
return (timestamp, DEACTIVATED, mode or self.mode, None)
def _mode_changed(self, timestamp, mode=None):
"""Creates a MODE_CHANGED event."""
return (timestamp, MODE_CHANGED, mode or self.mode, None)
def _validated(self, timestamp, is_active, created, mode=None, dump_duration_in_secs=300):
"""Creates a VALIDATED event."""
dump_end = timestamp
dump_start = add_microseconds(timestamp, int(dump_duration_in_secs) * -100000)
validation_info = {
'is_active': is_active,
'created': created,
'dump_start': dump_start,
'dump_end': dump_end,
}
return (timestamp, VALIDATED, mode or self.mode, validation_info)
def setUp(self):
self.initialize_ids()
fake_param = luigi.DateIntervalParameter()
self.task = CourseEnrollmentValidationTask(
interval=fake_param.parse('2013-12-17'),
output_root='/fake/output')
self.task.init_local()
self.user_id = 21
self.timestamp = "2013-12-17T15:38:32.805444"
self.mode = 'honor'
self.factory = SyntheticEventFactory(
timestamp=self.timestamp,
event_source='server',
event_type=ACTIVATED,
synthesizer='enrollment_from_db',
reason='db entry',
user_id=self.user_id,
course_id=self.course_id,
org_id=self.org_id,
)
def create_task(self, generate_before=True, tuple_output=True, include_nonstate_changes=True,
earliest_timestamp=None, expected_validation=None):
"""Create a task for testing purposes."""
interval = '2013-01-01-2014-10-10'
interval_value = luigi.DateIntervalParameter().parse(interval)
earliest_timestamp_value = luigi.DateHourParameter().parse(earliest_timestamp) if earliest_timestamp else None
expected_validation_value = (
luigi.DateHourParameter().parse(expected_validation) if expected_validation else None
)
self.task = CourseEnrollmentValidationTask(
interval=interval_value,
output_root="/fake/output",
generate_before=generate_before,
tuple_output=tuple_output,
include_nonstate_changes=include_nonstate_changes,
earliest_timestamp=earliest_timestamp_value,
expected_validation=expected_validation_value,
)
self.task.init_local()
def setUp(self):
self.initialize_ids()
fake_param = luigi.DateIntervalParameter()
self.task = CourseEnrollmentValidationTask(
interval=fake_param.parse('2013-12-17'),
output_root='/fake/output'
)
self.task.init_local()
self.user_id = 21
self.timestamp = "2013-12-17T15:38:32.805444"
self.mode = 'honor'
self.factory = SyntheticEventFactory(
timestamp=self.timestamp,
event_source='server',
event_type=ACTIVATED,
synthesizer='enrollment_from_db',
reason='db entry',
user_id=self.user_id,
course_id=self.course_id,
org_id=self.org_id,
)
class BaseCourseEnrollmentValidationTaskReducerTest(ReducerTestMixin,
unittest.TestCase):
"""Provide common methods for testing CourseEnrollmentValidationTask reducer."""
def setUp(self):
self.task_class = CourseEnrollmentValidationTask
super(BaseCourseEnrollmentValidationTaskReducerTest, self).setUp()
self.reduce_key = ('foo/bar/baz', 0)
self.mode = 'honor'
def create_validation_task(self,
generate_before=True,
tuple_output=True,
include_nonstate_changes=True,
earliest_timestamp=None,
expected_validation=None):
"""Create a task for testing purposes."""
interval = '2013-01-01-2014-10-10'
interval_value = luigi.DateIntervalParameter().parse(interval)
earliest_timestamp_value = luigi.DateHourParameter().parse(
earliest_timestamp) if earliest_timestamp else None
expected_validation_value = (
luigi.DateHourParameter().parse(expected_validation)
if expected_validation else None)
self.task = CourseEnrollmentValidationTask(
interval=interval_value,
output_root="/fake/output",
generate_before=generate_before,
tuple_output=tuple_output,
include_nonstate_changes=include_nonstate_changes,
earliest_timestamp=earliest_timestamp_value,
expected_validation=expected_validation_value,
)
self.task.init_local()
def _activated(self, timestamp, mode=None):
"""Creates an ACTIVATED event."""
return (timestamp, ACTIVATED, mode or self.mode, None)
def _deactivated(self, timestamp, mode=None):
"""Creates a DEACTIVATED event."""
return (timestamp, DEACTIVATED, mode or self.mode, None)
def _mode_changed(self, timestamp, mode=None):
"""Creates a MODE_CHANGED event."""
return (timestamp, MODE_CHANGED, mode or self.mode, None)
def _validated(self,
timestamp,
is_active,
created,
mode=None,
dump_duration_in_secs=300):
"""Creates a VALIDATED event."""
dump_end = timestamp
dump_start = add_microseconds(timestamp,
int(dump_duration_in_secs) * -100000)
validation_info = {
'is_active': is_active,
'created': created,
'dump_start': dump_start,
'dump_end': dump_end,
}
return (timestamp, VALIDATED, mode or self.mode, validation_info)
class CourseEnrollmentValidationTaskMapTest(InitializeOpaqueKeysMixin, unittest.TestCase):
"""
Tests to verify that event log parsing by mapper works correctly.
"""
def setUp(self):
self.initialize_ids()
fake_param = luigi.DateIntervalParameter()
self.task = CourseEnrollmentValidationTask(
interval=fake_param.parse('2013-12-17'),
output_root='/fake/output'
)
self.task.init_local()
self.user_id = 21
self.timestamp = "2013-12-17T15:38:32.805444"
self.mode = 'honor'
self.factory = SyntheticEventFactory(
timestamp=self.timestamp,
event_source='server',
event_type=ACTIVATED,
synthesizer='enrollment_from_db',
reason='db entry',
user_id=self.user_id,
course_id=self.course_id,
org_id=self.org_id,
)
def _create_event_log_line(self, **kwargs):
"""Create an event log with test values, as a JSON string."""
return json.dumps(self._create_event_dict(**kwargs))
def _create_event_dict(self, **kwargs):
"""Create an event log with test values, as a dict."""
event_data = {
'course_id': self.course_id,
'user_id': self.user_id,
'mode': self.mode,
}
event = self.factory.create_event_dict(event_data, **kwargs)
return event
def assert_no_output_for(self, line):
"""Assert that an input line generates no output."""
self.assertEquals(tuple(self.task.mapper(line)), tuple())
def test_non_enrollment_event(self):
line = 'this is garbage'
self.assert_no_output_for(line)
def test_unparseable_enrollment_event(self):
line = 'this is garbage but contains edx.course.enrollment'
self.assert_no_output_for(line)
def test_missing_event_type(self):
event_dict = self._create_event_dict()
event_dict['old_event_type'] = event_dict['event_type']
del event_dict['event_type']
line = json.dumps(event_dict)
self.assert_no_output_for(line)
def test_nonenroll_event_type(self):
line = self._create_event_log_line(event_type='edx.course.enrollment.unknown')
self.assert_no_output_for(line)
def test_bad_datetime(self):
line = self._create_event_log_line(time='this is a bogus time')
self.assert_no_output_for(line)
def test_bad_event_data(self):
line = self._create_event_log_line(event=["not an event"])
self.assert_no_output_for(line)
def test_illegal_course_id(self):
line = self._create_event_log_line(event={"course_id": ";;;;bad/id/val", "user_id": self.user_id})
self.assert_no_output_for(line)
def test_missing_user_id(self):
line = self._create_event_log_line(event={"course_id": self.course_id})
self.assert_no_output_for(line)
def test_good_enroll_event(self):
line = self._create_event_log_line()
event = tuple(self.task.mapper(line))
expected = (((self.course_id, self.user_id), (self.timestamp, ACTIVATED, self.mode, None)),)
self.assertEquals(event, expected)
def test_good_unenroll_event(self):
line = self._create_event_log_line(event_type=DEACTIVATED)
event = tuple(self.task.mapper(line))
expected = (((self.course_id, self.user_id), (self.timestamp, DEACTIVATED, self.mode, None)),)
self.assertEquals(event, expected)
def test_good_mode_change_event(self):
line = self._create_event_log_line(event_type=MODE_CHANGED)
event = tuple(self.task.mapper(line))
expected = (((self.course_id, self.user_id), (self.timestamp, MODE_CHANGED, self.mode, None)),)
self.assertEquals(event, expected)
def test_good_validation_event(self):
validation_info = {
'is_active': True,
'created': '2012-07-24T12:37:32.000000',
'dump_start': '2014-10-08T04:52:48.154228',
'dump_end': '2014-10-08T04:57:38.145282',
}
event_info = {
"course_id": self.course_id,
"user_id": self.user_id,
"mode": self.mode,
}
event_info.update(validation_info)
line = self._create_event_log_line(event_type=VALIDATED, event=event_info)
event = tuple(self.task.mapper(line))
expected = (((self.course_id, self.user_id), (self.timestamp, VALIDATED, self.mode, validation_info)),)
self.assertEquals(event, expected)
class BaseCourseEnrollmentValidationTaskReducerTest(unittest.TestCase):
"""Provide common methods for testing CourseEnrollmentValidationTask reducer."""
def setUp(self):
self.mode = 'honor'
@property
def key(self):
"""Returns key value to simulate output from mapper to pass to reducer."""
user_id = 0
course_id = 'foo/bar/baz'
return (course_id, user_id)
def create_task(self, generate_before=True, tuple_output=True, include_nonstate_changes=True,
earliest_timestamp=None, expected_validation=None):
"""Create a task for testing purposes."""
interval = '2013-01-01-2014-10-10'
interval_value = luigi.DateIntervalParameter().parse(interval)
earliest_timestamp_value = luigi.DateHourParameter().parse(earliest_timestamp) if earliest_timestamp else None
expected_validation_value = (
luigi.DateHourParameter().parse(expected_validation) if expected_validation else None
)
self.task = CourseEnrollmentValidationTask(
interval=interval_value,
output_root="/fake/output",
generate_before=generate_before,
tuple_output=tuple_output,
include_nonstate_changes=include_nonstate_changes,
earliest_timestamp=earliest_timestamp_value,
expected_validation=expected_validation_value,
)
self.task.init_local()
def _activated(self, timestamp, mode=None):
"""Creates an ACTIVATED event."""
return (timestamp, ACTIVATED, mode or self.mode, None)
def _deactivated(self, timestamp, mode=None):
"""Creates a DEACTIVATED event."""
return (timestamp, DEACTIVATED, mode or self.mode, None)
def _mode_changed(self, timestamp, mode=None):
"""Creates a MODE_CHANGED event."""
return (timestamp, MODE_CHANGED, mode or self.mode, None)
def _validated(self, timestamp, is_active, created, mode=None, dump_duration_in_secs=300):
"""Creates a VALIDATED event."""
dump_end = timestamp
dump_start = add_microseconds(timestamp, int(dump_duration_in_secs) * -100000)
validation_info = {
'is_active': is_active,
'created': created,
'dump_start': dump_start,
'dump_end': dump_end,
}
return (timestamp, VALIDATED, mode or self.mode, validation_info)
def _get_reducer_output(self, values):
"""Run reducer with provided values hardcoded key."""
return tuple(self.task.reducer(self.key, values))
def check_output(self, inputs, expected):
"""Compare generated with expected output."""
expected_with_key = tuple([(key, self.key + value) for key, value in expected])
self.assertEquals(self._get_reducer_output(inputs), expected_with_key)
class CourseEnrollmentValidationTaskMapTest(InitializeOpaqueKeysMixin,
unittest.TestCase):
"""
Tests to verify that event log parsing by mapper works correctly.
"""
def setUp(self):
self.initialize_ids()
fake_param = luigi.DateIntervalParameter()
self.task = CourseEnrollmentValidationTask(
interval=fake_param.parse('2013-12-17'),
output_root='/fake/output')
self.task.init_local()
self.user_id = 21
self.timestamp = "2013-12-17T15:38:32.805444"
self.mode = 'honor'
self.factory = SyntheticEventFactory(
timestamp=self.timestamp,
event_source='server',
event_type=ACTIVATED,
synthesizer='enrollment_from_db',
reason='db entry',
user_id=self.user_id,
course_id=self.course_id,
org_id=self.org_id,
)
def _create_event_log_line(self, **kwargs):
"""Create an event log with test values, as a JSON string."""
return json.dumps(self._create_event_dict(**kwargs))
def _create_event_dict(self, **kwargs):
"""Create an event log with test values, as a dict."""
event_data = {
'course_id': self.course_id,
'user_id': self.user_id,
'mode': self.mode,
}
event = self.factory.create_event_dict(event_data, **kwargs)
return event
def assert_no_output_for(self, line):
"""Assert that an input line generates no output."""
self.assertEquals(tuple(self.task.mapper(line)), tuple())
def test_non_enrollment_event(self):
line = 'this is garbage'
self.assert_no_output_for(line)
def test_unparseable_enrollment_event(self):
line = 'this is garbage but contains edx.course.enrollment'
self.assert_no_output_for(line)
def test_missing_event_type(self):
event_dict = self._create_event_dict()
event_dict['old_event_type'] = event_dict['event_type']
del event_dict['event_type']
line = json.dumps(event_dict)
self.assert_no_output_for(line)
def test_nonenroll_event_type(self):
line = self._create_event_log_line(
event_type='edx.course.enrollment.unknown')
self.assert_no_output_for(line)
def test_bad_datetime(self):
line = self._create_event_log_line(time='this is a bogus time')
self.assert_no_output_for(line)
def test_bad_event_data(self):
line = self._create_event_log_line(event=["not an event"])
self.assert_no_output_for(line)
def test_illegal_course_id(self):
line = self._create_event_log_line(event={
"course_id": ";;;;bad/id/val",
"user_id": self.user_id
})
self.assert_no_output_for(line)
def test_missing_user_id(self):
line = self._create_event_log_line(event={"course_id": self.course_id})
self.assert_no_output_for(line)
def test_good_enroll_event(self):
line = self._create_event_log_line()
event = tuple(self.task.mapper(line))
expected = (((self.course_id, self.user_id),
(self.timestamp, ACTIVATED, self.mode, None)), )
self.assertEquals(event, expected)
def test_good_unenroll_event(self):
line = self._create_event_log_line(event_type=DEACTIVATED)
event = tuple(self.task.mapper(line))
expected = (((self.course_id, self.user_id),
(self.timestamp, DEACTIVATED, self.mode, None)), )
self.assertEquals(event, expected)
def test_good_mode_change_event(self):
line = self._create_event_log_line(event_type=MODE_CHANGED)
event = tuple(self.task.mapper(line))
expected = (((self.course_id, self.user_id),
(self.timestamp, MODE_CHANGED, self.mode, None)), )
self.assertEquals(event, expected)
def test_good_validation_event(self):
validation_info = {
'is_active': True,
'created': '2012-07-24T12:37:32.000000',
'dump_start': '2014-10-08T04:52:48.154228',
'dump_end': '2014-10-08T04:57:38.145282',
}
event_info = {
"course_id": self.course_id,
"user_id": self.user_id,
"mode": self.mode,
}
event_info.update(validation_info)
line = self._create_event_log_line(event_type=VALIDATED,
event=event_info)
event = tuple(self.task.mapper(line))
expected = (((self.course_id, self.user_id),
(self.timestamp, VALIDATED, self.mode,
validation_info)), )
self.assertEquals(event, expected)
class BaseCourseEnrollmentValidationTaskReducerTest(unittest.TestCase):
"""Provide common methods for testing CourseEnrollmentValidationTask reducer."""
def setUp(self):
self.mode = 'honor'
@property
def key(self):
"""Returns key value to simulate output from mapper to pass to reducer."""
user_id = 0
course_id = 'foo/bar/baz'
return (course_id, user_id)
def create_task(self,
generate_before=True,
tuple_output=True,
include_nonstate_changes=True,
earliest_timestamp=None,
expected_validation=None):
"""Create a task for testing purposes."""
interval = '2013-01-01-2014-10-10'
interval_value = luigi.DateIntervalParameter().parse(interval)
earliest_timestamp_value = luigi.DateHourParameter().parse(
earliest_timestamp) if earliest_timestamp else None
expected_validation_value = (
luigi.DateHourParameter().parse(expected_validation)
if expected_validation else None)
self.task = CourseEnrollmentValidationTask(
interval=interval_value,
output_root="/fake/output",
generate_before=generate_before,
tuple_output=tuple_output,
include_nonstate_changes=include_nonstate_changes,
earliest_timestamp=earliest_timestamp_value,
expected_validation=expected_validation_value,
)
self.task.init_local()
def _activated(self, timestamp, mode=None):
"""Creates an ACTIVATED event."""
return (timestamp, ACTIVATED, mode or self.mode, None)
def _deactivated(self, timestamp, mode=None):
"""Creates a DEACTIVATED event."""
return (timestamp, DEACTIVATED, mode or self.mode, None)
def _mode_changed(self, timestamp, mode=None):
"""Creates a MODE_CHANGED event."""
return (timestamp, MODE_CHANGED, mode or self.mode, None)
def _validated(self,
timestamp,
is_active,
created,
mode=None,
dump_duration_in_secs=300):
"""Creates a VALIDATED event."""
dump_end = timestamp
dump_start = add_microseconds(timestamp,
int(dump_duration_in_secs) * -100000)
validation_info = {
'is_active': is_active,
'created': created,
'dump_start': dump_start,
'dump_end': dump_end,
}
return (timestamp, VALIDATED, mode or self.mode, validation_info)
def _get_reducer_output(self, values):
"""Run reducer with provided values hardcoded key."""
return tuple(self.task.reducer(self.key, values))
def check_output(self, inputs, expected):
"""Compare generated with expected output."""
expected_with_key = tuple([(key, self.key + value)
for key, value in expected])
self.assertEquals(self._get_reducer_output(inputs), expected_with_key)