class CreateEnrollmentValidationEventsTask(MultiOutputMapReduceJobTask):
"""
Convert a database dump of course enrollment into log files of validation events.
Read from a directory location that points to a Sqoop dump of student_courseenrollment
table. Use map reduce simply because it allows the multiple file output to be read
uniformly. But it allows us to also separate the enrollment results into separate
courses so that validation runs can be more fine-grained.
The date for the synthesized events is the end time of the Sqoop dump. This
is when the particular enrollment states were observed.
Parameters:
source_dir: the URL of the location of the desired database dump. This should
include the 'dt=<date>' partition specification.
Other parameters are defined by MultiOutputMapReduceJobTask.
"""
# Note: we could just read the corresponding validation data into
# the reducer. So this would just need to produce reducer input
# instead of mapper input. Problem with that is that if there
# were courses for which there were database entries but no
# events, they wouldn't get validated. So we put the events into
# the mapper to make sure all courses get processed.
# This defines the directory (with the dt=<date> partition) that contains
# the desired database dump.
source_dir = luigi.Parameter()
def requires_hadoop(self):
# Check first if running locally with Sqoop output.
target = get_target_from_url(self.source_dir)
if isinstance(target, luigi.LocalTarget) and os.path.isdir(self.source_dir):
files = [f for f in os.listdir(self.source_dir) if f.startswith("part")]
for filename in files:
yield ExternalURL(url_path_join(self.source_dir, filename))
else:
yield ExternalURL(self.source_dir)
def init_local(self):
super(CreateEnrollmentValidationEventsTask, self).init_local()
# need to determine the date of the input, by reading the appropriate
# metadata file. File looks like this:
# {"start_time": "2014-10-08T04:52:48.154228", "end_time": "2014-10-08T04:55:18.269070"}
metadata_target = self._get_metadata_target()
with metadata_target.open('r') as metadata_file:
metadata = json.load(metadata_file)
self.dump_start_time = metadata["start_time"]
self.dump_end_time = metadata["end_time"]
log.debug("Found self.dump_start_time = %s end_time = %s", self.dump_start_time, self.dump_end_time)
self.dump_date = ''.join((self.dump_start_time.split('T')[0]).split('-'))
# Set the timestamp of all events to be the dump's end time.
# The events that are actually dumped are not within a transaction,
# so the actual event time may be earlier, anywhere up to the dump's start time.
self.factory = SyntheticEventFactory(
timestamp=self.dump_end_time,
event_source='server',
event_type=VALIDATED,
synthesizer='enrollment_from_db',
reason='db entry'
)
def _get_metadata_target(self):
"""Returns target for metadata file from the given dump."""
# find the .metadata file in the source directory.
metadata_path = url_path_join(self.source_dir, ".metadata")
return get_target_from_url(metadata_path)
def mapper(self, line):
fields = line.split('\x01')
if len(fields) != 6:
log.error("Encountered bad input: %s", line)
return
(_db_id, user_id_string, encoded_course_id, mysql_created, mysql_is_active, mode) = fields
# `created` is of the form '2012-07-25 12:26:22.0', coming out of
# mysql. Convert it to isoformat.
created = mysql_datetime_to_isoformat(mysql_created)
# `is_active` should be a boolean and `user_id` is an int.
is_active = (mysql_is_active == "true")
user_id = int(user_id_string)
# Note that we do not have several standard properties that we
# might expect in such an event. These include a username,
# host, session_id, agent. These values will be stubbed by
# the factory as empty strings.
course_id = encoded_course_id.decode('utf-8')
# data for the particular type of event:
event_data = {
'course_id': course_id,
'user_id': user_id,
'mode': mode,
'is_active': is_active,
'created': created,
'dump_start': self.dump_start_time,
'dump_end': self.dump_end_time,
}
# stuff for context:
event_properties = {
'user_id': user_id,
'course_id': course_id,
'org_id': opaque_key_util.get_org_id_for_course(course_id),
}
event = self.factory.create_event(event_data, **event_properties)
# Use the original utf-8 version of the course_id as the key.
# (Note that if we want everything zipped into a single file,
# then we can just pass a single dummy value for the key instead of
# breaking the output out by course_id.)
yield encoded_course_id, event
def multi_output_reducer(self, _key, values, output_file):
with gzip.GzipFile(mode='wb', fileobj=output_file) as outfile:
for value in values:
outfile.write(value)
outfile.write('\n')
def output_path_for_key(self, course_id):
filename_safe_course_id = opaque_key_util.get_filename_safe_course_id(course_id, '_')
filename = u'{course_id}_enroll_validated_{dumpdate}.log.gz'.format(
course_id=filename_safe_course_id,
dumpdate=self.dump_date,
)
return url_path_join(self.output_root, filename)
