def mapper(self, line):
value = self.get_event_and_date_string(line)
if value is None:
return
event, date_string = value
username = eventlog.get_event_username(event)
if not username:
return
# Get timestamp instead of date string, so we get the latest ip
# address for events on the same day.
timestamp = eventlog.get_event_time_string(event)
if not timestamp:
return
ip_address = event.get('ip')
if not ip_address:
log.warning("No ip_address found for user '%s' on '%s'.", username,
timestamp)
return
# Get the course_id from context, if it happens to be present.
# It's okay if it isn't.
# (Not sure if there are particular types of course
# interaction we care about, but we might want to only collect
# the course_id off of explicit events, and ignore implicit
# events as not being "real" interactions with course content.
# Or maybe we add a flag indicating explicit vs. implicit, so
# that this can be better teased apart. For example, we could
# use the latest explicit event for a course, but if there are
# none, then use the latest implicit event for the course, and
# if there are none, then use the latest overall event.)
course_id = eventlog.get_course_id(event)
# For multi-output, we will generate a single file for each key value.
# When looking at location for user in a course, we don't want to have
# an output file per course per date, so just use date as the key,
# and have a single file representing all events on the date.
yield date_string, (timestamp, ip_address, course_id, username)
def mapper(self, line):
value = self.get_event_and_date_string(line)
if value is None:
return
event, date_string = value
username = eventlog.get_event_username(event)
if not username:
log.error("Encountered event with no username: %s", event)
self.incr_counter('Active Users last year', 'Discard Event Missing username', 1)
return
date = datetime.date(*[int(x) for x in date_string.split('-')])
iso_year, iso_weekofyear, _iso_weekday = date.isocalendar()
week = isoweek.Week(iso_year, iso_weekofyear)
start_date = week.monday().isoformat()
end_date = (week.sunday() + datetime.timedelta(1)).isoformat()
yield (start_date, end_date, username), 1
def mapper(self, line):
value = self.get_event_and_date_string(line)
if value is None:
return
event, date_string = value
username = eventlog.get_event_username(event)
if not username:
return
# Get timestamp instead of date string, so we get the latest ip
# address for events on the same day.
timestamp = eventlog.get_event_time_string(event)
if not timestamp:
return
ip_address = event.get('ip')
if not ip_address:
log.warning("No ip_address found for user '%s' on '%s'.", username, timestamp)
return
# Get the course_id from context, if it happens to be present.
# It's okay if it isn't.
# (Not sure if there are particular types of course
# interaction we care about, but we might want to only collect
# the course_id off of explicit events, and ignore implicit
# events as not being "real" interactions with course content.
# Or maybe we add a flag indicating explicit vs. implicit, so
# that this can be better teased apart. For example, we could
# use the latest explicit event for a course, but if there are
# none, then use the latest implicit event for the course, and
# if there are none, then use the latest overall event.)
course_id = eventlog.get_course_id(event)
# For multi-output, we will generate a single file for each key value.
# When looking at location for user in a course, we don't want to have
# an output file per course per date, so just use date as the key,
# and have a single file representing all events on the date.
yield date_string, (timestamp, ip_address, course_id, username)
def test_event_username_with_trailing_whitespace(self):
item = {"username": "******"}
self.assertEquals(eventlog.get_event_username(item), u'bub')
def test_empty_event_username(self):
item = {"username": "******"}
self.assertIsNone(eventlog.get_event_username(item))
def test_missing_event_username(self):
item = {"something else": "not an event"}
self.assertIsNone(eventlog.get_event_username(item))
def _remap_user_info_in_event(self, event, event_data):
"""
Harvest user info from event, and remap those values (in place) where appropriate.
Returns a dict of iterables, with key values of 'username', 'user_id', and 'name'.
"""
# Find user info, and
debug_str = self._get_log_string_for_event(event)
# Create a user_info structure to collect relevant user information to look
# for elsewhere in the event. We need to return a dictionary of iterables,
# but since we will potentially be adding the same values repeatedly from
# different parts of the event, a set will make sure these are deduped.
user_info = defaultdict(set)
# Note that eventlog.get_event_username() does a strip on the username and checks for zero-len,
# so we don't have to do so here.
username = eventlog.get_event_username(event)
if username is not None:
username = username.decode('utf8')
remapped_username = self._remap_username(username, user_info)
if remapped_username is not None:
event['username'] = remapped_username
else:
log.error(
"Redacting unrecognized username for '%s' field: '%s' %s",
'username', username, debug_str)
event['username'] = REDACTED_USERNAME
# Get the user_id from context, either as an int or None, and remap.
user_id = self._get_user_id_as_int(
event.get('context', {}).get('user_id'))
if user_id is not None:
user_info['user_id'].add(user_id)
info = self._get_user_info_for_user_id(user_id)
if info is not None:
for key, value in info.iteritems():
user_info[key].add(value)
if username is not None and 'username' in info and username != info[
'username']:
log.error(
u"user_id ('%s'=>'%s') does not match username ('%s') %s",
user_id,
info['username'],
username,
debug_str,
)
event['context']['user_id'] = self.remap_id(user_id)
# Clean username from context.
if 'context' in event:
# Remap value of username in context, if it is present. (Removed in more recent events.)
if 'username' in event['context'] and len(
event['context']['username'].strip()) > 0:
context_username = event['context']['username'].strip().decode(
'utf8')
remapped_username = self._remap_username(
context_username, user_info)
if remapped_username is not None:
event['context']['username'] = remapped_username
else:
log.error(
"Redacting unrecognized username for '%s' field: '%s' %s",
'context.username', context_username, debug_str)
event['context']['username'] = REDACTED_USERNAME
# Look into the event payload.
if event_data:
# Get the user_id from payload and remap.
event_user_id = self._get_user_id_as_int(event_data.get('user_id'))
if event_user_id is not None:
user_info['user_id'].add(event_user_id)
info = self._get_user_info_for_user_id(event_user_id)
if info is not None:
for key, value in info.iteritems():
user_info[key].add(value)
event_data['user_id'] = self.remap_id(event_user_id)
# Remap values of usernames in payload, if present. Usernames may appear with different key values.
# TODO: confirm that these values are usernames, not user_id values. (User_id values will fail remapping.)
for username_key in ['username', 'instructor', 'student', 'user']:
if username_key in event_data and len(
event_data[username_key].strip()) > 0:
event_username = event_data[username_key].strip().decode(
'utf8')
remapped_username = self._remap_username(
event_username, user_info)
if remapped_username is not None:
event_data[username_key] = remapped_username
else:
log.error(
"Redacting unrecognized username for 'event.%s' field: '%s' %s",
username_key, event_username, debug_str)
event_data[username_key] = REDACTED_USERNAME
# Finally return the fully-constructed dict.
return user_info
def get_userinfo_from_event(self, event, event_data):
# Start simply, and just get obvious info. See what it matches.
# Need to check back on this, but we really only need to know
# if this information is wrong. What we want to come out
# of this is a user_id and/or a username that can be used for
# cleaning the rest of the event.
# And actually, what we also need is the relevant fullname to use,
# so we need to pick entries out of the user_info that match.
# One or more? No analysis was really made of alignment.
# So we'll have to do it here...
event_type = event.get('event_type')
if isinstance(event_type, str):
event_type = event_type.decode('utf8')
debug_str = u" [event_type='{}']".format(event_type)
username_entry = None
username = eventlog.get_event_username(event)
if username is not None:
username = username.decode('utf8')
if self.user_info is not None:
username_entry = self.user_info.get(username)
if username_entry is None:
log.error(u"username ('%s') is unknown to user_info %s",
username, debug_str)
# Get the user_id either as an int or None
userid_entry = None
user_id = self.get_user_id_as_int(
event.get('context', {}).get('user_id'))
if user_id is not None:
if self.user_info is not None:
userid_entry = self.user_info.get(user_id)
if userid_entry is None:
log.error(u"user_id ('%s') is unknown to user_info %s",
user_id, debug_str)
elif username_entry and userid_entry != username_entry:
log.error(
u"user_id ('%s'='%s') does not match username ('%s'='%s') %s",
userid_entry.get('user_id'),
userid_entry.get('username'),
username_entry.get('username'),
username_entry.get('user_id'),
debug_str,
)
event_userid_entry = None
if event_data and isinstance(event_data, dict):
event_user_id = self.get_user_id_as_int(event_data.get('user_id'))
if event_user_id:
if self.user_info is not None:
event_userid_entry = self.user_info.get(event_user_id)
if event_userid_entry is None:
log.error(
u"Event_user_id ('%s') is unknown to user_info %s",
event_user_id, debug_str)
if user_id is None:
# This is way too common. In testing, every edx.course.enrollment.xxx had the user_id in the event but not
# in context. Weird.
# log.warning(u"Found user_id ('%s') in event but nothing in context %s", event_user_id, debug_str)
pass
elif event_userid_entry and userid_entry != event_userid_entry:
# This turns out to be somewhat expected for certain event types where one user is doing something on behalf
# of another user. The actor is in context, and the object is in event payload.
if event_type not in EVENT_TYPES_WITH_DIFFERENT_USERIDS:
log.error(
u"Context user_id ('%s'='%s') does not match event user_id ('%s'='%s') %s",
userid_entry.get('user_id'),
userid_entry.get('username'),
event_userid_entry.get('username'),
event_userid_entry.get('user_id'),
debug_str,
)
elif event_user_id != user_id:
log.error(
u"Found user_id ('%s') in event that was different from context ('%s') %s",
event_user_id, user_id, debug_str)
# We choose the event user_id over the context, and fall back on the username.
if event_userid_entry is not None:
return event_userid_entry
elif userid_entry is not None:
return userid_entry
else:
return username_entry
def _remap_user_info_in_event(self, event, event_data):
"""
Harvest user info from event, and remap those values (in place) where appropriate.
Returns a dict of iterables, with key values of 'username', 'user_id', and 'name'.
"""
# Find user info, and
debug_str = self._get_log_string_for_event(event)
# Create a user_info structure to collect relevant user information to look
# for elsewhere in the event. We need to return a dictionary of iterables,
# but since we will potentially be adding the same values repeatedly from
# different parts of the event, a set will make sure these are deduped.
user_info = defaultdict(set)
# Note that eventlog.get_event_username() does a strip on the username and checks for zero-len,
# so we don't have to do so here.
username = eventlog.get_event_username(event)
if username is not None:
username = username.decode('utf8')
remapped_username = self._remap_username(username, user_info)
if remapped_username is not None:
event['username'] = remapped_username
else:
log.error("Redacting unrecognized username for '%s' field: '%s' %s", 'username', username, debug_str)
event['username'] = REDACTED_USERNAME
# Get the user_id from context, either as an int or None, and remap.
user_id = self._get_user_id_as_int(event.get('context', {}).get('user_id'))
if user_id is not None:
user_info['user_id'].add(user_id)
info = self._get_user_info_for_user_id(user_id)
if info is not None:
for key, value in info.iteritems():
user_info[key].add(value)
if username is not None and 'username' in info and username != info['username']:
log.error(
u"user_id ('%s'=>'%s') does not match username ('%s') %s",
user_id, info['username'], username, debug_str,
)
event['context']['user_id'] = self.remap_id(user_id)
# Clean username from context.
if 'context' in event:
# Remap value of username in context, if it is present. (Removed in more recent events.)
if 'username' in event['context'] and len(event['context']['username'].strip()) > 0:
context_username = event['context']['username'].strip().decode('utf8')
remapped_username = self._remap_username(context_username, user_info)
if remapped_username is not None:
event['context']['username'] = remapped_username
else:
log.error("Redacting unrecognized username for '%s' field: '%s' %s", 'context.username',
context_username, debug_str)
event['context']['username'] = REDACTED_USERNAME
# Look into the event payload.
if event_data:
# Get the user_id from payload and remap.
event_user_id = self._get_user_id_as_int(event_data.get('user_id'))
if event_user_id is not None:
user_info['user_id'].add(event_user_id)
info = self._get_user_info_for_user_id(event_user_id)
if info is not None:
for key, value in info.iteritems():
user_info[key].add(value)
event_data['user_id'] = self.remap_id(event_user_id)
# Remap values of usernames in payload, if present. Usernames may appear with different key values.
# TODO: confirm that these values are usernames, not user_id values. (User_id values will fail remapping.)
for username_key in ['username', 'instructor', 'student', 'user']:
if username_key in event_data and len(event_data[username_key].strip()) > 0:
event_username = event_data[username_key].strip().decode('utf8')
remapped_username = self._remap_username(event_username, user_info)
if remapped_username is not None:
event_data[username_key] = remapped_username
else:
log.error("Redacting unrecognized username for 'event.%s' field: '%s' %s",
username_key, event_username, debug_str)
event_data[username_key] = REDACTED_USERNAME
# Finally return the fully-constructed dict.
return user_info
def get_userinfo_from_event(self, event, event_data):
# Start simply, and just get obvious info. See what it matches.
# Need to check back on this, but we really only need to know
# if this information is wrong. What we want to come out
# of this is a user_id and/or a username that can be used for
# cleaning the rest of the event.
# And actually, what we also need is the relevant fullname to use,
# so we need to pick entries out of the user_info that match.
# One or more? No analysis was really made of alignment.
# So we'll have to do it here...
event_type = event.get('event_type')
if isinstance(event_type, str):
event_type = event_type.decode('utf8')
debug_str = u" [event_type='{}']".format(event_type)
username_entry = None
username = eventlog.get_event_username(event)
if username is not None:
username = username.decode('utf8')
if self.user_info is not None:
username_entry = self.user_info.get(username)
if username_entry is None:
log.error(u"username ('%s') is unknown to user_info %s", username, debug_str)
# Get the user_id either as an int or None
userid_entry = None
user_id = self.get_user_id_as_int(event.get('context', {}).get('user_id'))
if user_id is not None:
if self.user_info is not None:
userid_entry = self.user_info.get(user_id)
if userid_entry is None:
log.error(u"user_id ('%s') is unknown to user_info %s", user_id, debug_str)
elif username_entry and userid_entry != username_entry:
log.error(
u"user_id ('%s'='%s') does not match username ('%s'='%s') %s",
userid_entry.get('user_id'), userid_entry.get('username'), username_entry.get('username'), username_entry.get('user_id'), debug_str,
)
event_userid_entry = None
if event_data and isinstance(event_data, dict):
event_user_id = self.get_user_id_as_int(event_data.get('user_id'))
if event_user_id:
if self.user_info is not None:
event_userid_entry = self.user_info.get(event_user_id)
if event_userid_entry is None:
log.error(u"Event_user_id ('%s') is unknown to user_info %s", event_user_id, debug_str)
if user_id is None:
# This is way too common. In testing, every edx.course.enrollment.xxx had the user_id in the event but not
# in context. Weird.
# log.warning(u"Found user_id ('%s') in event but nothing in context %s", event_user_id, debug_str)
pass
elif event_userid_entry and userid_entry != event_userid_entry:
# This turns out to be somewhat expected for certain event types where one user is doing something on behalf
# of another user. The actor is in context, and the object is in event payload.
if event_type not in EVENT_TYPES_WITH_DIFFERENT_USERIDS:
log.error(
u"Context user_id ('%s'='%s') does not match event user_id ('%s'='%s') %s",
userid_entry.get('user_id'), userid_entry.get('username'), event_userid_entry.get('username'), event_userid_entry.get('user_id'), debug_str,
)
elif event_user_id != user_id:
log.error(u"Found user_id ('%s') in event that was different from context ('%s') %s", event_user_id, user_id, debug_str)
# We choose the event user_id over the context, and fall back on the username.
if event_userid_entry is not None:
return event_userid_entry
elif userid_entry is not None:
return userid_entry
else:
return username_entry
def test_event_username_with_trailing_whitespace(self):
item = {"username": "******"}
self.assertEquals(eventlog.get_event_username(item), u'bub')
def test_empty_event_username(self):
item = {"username": "******"}
self.assertIsNone(eventlog.get_event_username(item))
def test_missing_event_username(self):
item = {"something else": "not an event"}
self.assertIsNone(eventlog.get_event_username(item))