def server_track(request, event_type, event, page=None):
"""
Log events related to server requests.
Handle the situation where the request may be NULL, as may happen with management commands.
"""
if event_type.startswith("/event_logs") and request.user.is_staff:
return # don't log
try:
username = request.user.username
except:
username = "******"
# define output:
event = {
"username": username,
"ip": _get_request_header(request, 'REMOTE_ADDR'),
"event_source": "server",
"event_type": event_type,
"event": event,
"agent": _get_request_header(request, 'HTTP_USER_AGENT'),
"page": page,
"time": datetime.datetime.utcnow(),
"host": _get_request_header(request, 'SERVER_NAME'),
"context": eventtracker.get_tracker().resolve_context(),
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
log_event(event)
def server_track(request, event_type, event, page=None):
"""
Log events related to server requests.
Handle the situation where the request may be NULL, as may happen with management commands.
"""
if event_type.startswith("/event_logs") and request.user.is_staff:
return # don't log
try:
username = request.user.username
except:
username = "******"
# define output:
event = {
"username": username,
"ip": _get_request_ip(request),
"referer": _get_request_header(request, 'HTTP_REFERER'),
"accept_language": _get_request_header(request, 'HTTP_ACCEPT_LANGUAGE'),
"event_source": "server",
"event_type": event_type,
"event": event,
"agent": _get_request_header(request, 'HTTP_USER_AGENT').decode('latin1'),
"page": page,
"time": datetime.datetime.utcnow().replace(tzinfo=pytz.utc),
"host": _get_request_header(request, 'SERVER_NAME'),
"context": eventtracker.get_tracker().resolve_context(),
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
log_event(event)
def user_track(request):
"""
Log when POST call to "event" URL is made by a user. Uses request.REQUEST
to allow for GET calls.
GET or POST call should provide "event_type", "event", and "page" arguments.
"""
try: # TODO: Do the same for many of the optional META parameters
username = request.user.username
except:
username = "******"
page = _get_request_value(request, 'page')
with eventtracker.get_tracker().context(
'edx.course.browser', contexts.course_context_from_url(page)):
context = eventtracker.get_tracker().resolve_context()
event = {
"username": username,
"session": context.get('session', ''),
"ip": _get_request_header(request, 'REMOTE_ADDR'),
"referer": _get_request_header(request, 'HTTP_REFERER'),
"accept_language": _get_request_header(request,
'HTTP_ACCEPT_LANGUAGE'),
"event_source": "browser",
"event_type": _get_request_value(request, 'event_type'),
"event": _get_request_value(request, 'event'),
"agent": _get_request_header(request, 'HTTP_USER_AGENT'),
"page": page,
"time": datetime.datetime.utcnow(),
"host": _get_request_header(request, 'SERVER_NAME'),
"context": context,
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
log_event(event)
return HttpResponse('success')
def server_track(request, event_type, event, page=None):
"""
Log events related to server requests.
Handle the situation where the request may be NULL, as may happen with management commands.
"""
if event_type.startswith("/event_logs") and request.user.is_staff:
return # don't log
try:
username = request.user.username
except:
username = "******"
host_header = 'HTTP_HOST' if request is not None and 'HTTP_HOST' in request.META else 'SERVER_NAME'
# define output:
event = {
"username": username,
"ip": _get_request_ip(request),
"referer": _get_request_header(request, 'HTTP_REFERER'),
"accept_language": _get_request_header(request,
'HTTP_ACCEPT_LANGUAGE'),
"event_source": "server",
"event_type": event_type,
"event": event,
"agent": _get_request_header(request,
'HTTP_USER_AGENT').decode('latin1'),
"page": page,
"time": datetime.datetime.utcnow().replace(tzinfo=pytz.utc),
"host": _get_request_header(request, host_header),
"context": eventtracker.get_tracker().resolve_context(),
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
log_event(event)
def user_track(request):
"""
Log when POST call to "event" URL is made by a user. Uses request.REQUEST
to allow for GET calls.
GET or POST call should provide "event_type", "event", and "page" arguments.
"""
try: # TODO: Do the same for many of the optional META parameters
username = request.user.username
except:
username = "******"
page = _get_request_value(request, 'page')
with eventtracker.get_tracker().context('edx.course.browser', contexts.course_context_from_url(page)):
context = eventtracker.get_tracker().resolve_context()
event = {
"username": username,
"session": context.get('session', ''),
"ip": _get_request_header(request, 'REMOTE_ADDR'),
"referer": _get_request_header(request, 'HTTP_REFERER'),
"accept_language": _get_request_header(request, 'HTTP_ACCEPT_LANGUAGE'),
"event_source": "browser",
"event_type": _get_request_value(request, 'event_type'),
"event": _get_request_value(request, 'event'),
"agent": _get_request_header(request, 'HTTP_USER_AGENT'),
"page": page,
"time": datetime.datetime.utcnow(),
"host": _get_request_header(request, 'SERVER_NAME'),
"context": context,
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
log_event(event)
return HttpResponse('success')
def track_segmentio_event(request):
"""
An endpoint for logging events using segment.io's webhook integration.
segment.io provides a custom integration mechanism that initiates a request to a configurable URL every time an
event is received by their system. This endpoint is designed to receive those requests and convert the events into
standard tracking log entries.
For now we limit the scope of handled events to track and screen events from mobile devices. In the future we could
enable logging of other types of events, however, there is significant overlap with our non-segment.io based event
tracking. Given that segment.io is closed third party solution we are limiting its required usage to just
collecting events from mobile devices for the time being.
Many of the root fields of a standard edX tracking event are read out of the "properties" dictionary provided by the
segment.io event, which is, in turn, provided by the client that emitted the event.
In order for an event to be logged the following preconditions must be met:
* The "key" query string parameter must exactly match the django setting TRACKING_SEGMENTIO_WEBHOOK_SECRET. While
the endpoint is public, we want to limit access to it to the segment.io servers only.
* The value of the "channel" field of the event must be included in the list specified by the django setting
TRACKING_SEGMENTIO_ALLOWED_CHANNELS. This is intended to restrict the set of events to specific channels. For
example: just mobile devices.
* The value of the "action" field of the event must be included in the list specified by the django setting
TRACKING_SEGMENTIO_ALLOWED_ACTIONS. In order to make use of *all* of the features segment.io offers we would have
to implement some sort of persistent storage of information contained in some actions (like identify). For now,
we defer support of those actions and just support a limited set that can be handled without storing information
in external state.
* The value of the standard "userId" field of the event must be an integer that can be used to look up the user
using the primary key of the User model.
* Include an "event_type" field in the properties dictionary that indicates the edX event type. Note this can differ
from the "event" field found in the root of a segment.io event. The "event" field at the root of the structure is
intended to be human readable, the "event_type" field is expected to conform to the standard for naming events
found in the edX data documentation.
Additionally the event can optionally:
* Provide a "context" dictionary in the properties dictionary. This dictionary will be applied to the
existing context on the server overriding any existing keys. This context dictionary should include a "course_id"
field when the event is scoped to a particular course. The value of this field should be a valid course key. The
context may contain other arbitrary data that will be logged with the event, for example: identification
information for the device that emitted the event.
* Provide a "page" parameter in the properties dictionary which indicates the page that was being displayed to the
user or the mobile application screen that was visible to the user at the time the event was emitted.
"""
# Validate the security token. We must use a query string parameter for this since we cannot customize the POST body
# in the segment.io webhook configuration, we can only change the URL that they call, so we force this token to be
# included in the URL and reject any requests that do not include it. This also assumes HTTPS is used to make the
# connection between their server and ours.
expected_secret = getattr(settings, 'TRACKING_SEGMENTIO_WEBHOOK_SECRET', None)
provided_secret = request.GET.get('key')
if not expected_secret or provided_secret != expected_secret:
return failure_response(ERROR_UNAUTHORIZED, status=401)
# The POST body will contain the JSON encoded event
full_segment_event = request.json
def logged_failure_response(*args, **kwargs):
"""Indicate a failure and log information about the event that will aide debugging efforts"""
failed_response = failure_response(*args, **kwargs)
log.warning('Unable to process event received from segment.io: %s', json.dumps(full_segment_event))
return failed_response
# Selectively listen to particular channels
channel = full_segment_event.get('channel')
allowed_channels = [c.lower() for c in getattr(settings, 'TRACKING_SEGMENTIO_ALLOWED_CHANNELS', [])]
if not channel or channel.lower() not in allowed_channels:
return response(WARNING_IGNORED_CHANNEL, committed=False)
# Ignore actions that are unsupported
action = full_segment_event.get('action')
allowed_actions = [a.lower() for a in getattr(settings, 'TRACKING_SEGMENTIO_ALLOWED_ACTIONS', [])]
if not action or action.lower() not in allowed_actions:
return response(WARNING_IGNORED_ACTION, committed=False)
# We mostly care about the properties
segment_event = full_segment_event.get('properties', {})
context = {}
# Start with the context provided by segment.io in the "client" field if it exists
segment_context = full_segment_event.get('context')
if segment_context:
context['client'] = segment_context
# Overlay any context provided in the properties
context.update(segment_event.get('context', {}))
user_id = full_segment_event.get('userId')
if not user_id:
return logged_failure_response(ERROR_MISSING_USER_ID)
# userId is assumed to be the primary key of the django User model
try:
user = User.objects.get(pk=user_id)
except User.DoesNotExist:
return logged_failure_response(ERROR_USER_NOT_EXIST)
except ValueError:
return logged_failure_response(ERROR_INVALID_USER_ID)
else:
context['user_id'] = user_id
# course_id is expected to be provided in the context when applicable
course_id = context.get('course_id')
if course_id:
try:
course_key = CourseKey.from_string(course_id)
context['org_id'] = course_key.org
except InvalidKeyError:
log.warning(
'unable to parse course_id "{course_id}" from event: {event}'.format(
course_id=course_id,
event=json.dumps(full_segment_event),
),
exc_info=True
)
if 'timestamp' in full_segment_event:
time = parse_iso8601_timestamp(full_segment_event['timestamp'])
else:
return logged_failure_response(ERROR_MISSING_TIMESTAMP)
if 'receivedAt' in full_segment_event:
context['received_at'] = parse_iso8601_timestamp(full_segment_event['receivedAt'])
else:
return logged_failure_response(ERROR_MISSING_RECEIVED_AT)
if 'event_type' in segment_event:
event_type = segment_event['event_type']
else:
return logged_failure_response(ERROR_MISSING_EVENT_TYPE)
with eventtracker.get_tracker().context('edx.segmentio', context):
complete_context = eventtracker.get_tracker().resolve_context()
event = {
"username": user.username,
"event_type": event_type,
# Will be either "mobile", "browser" or "server". These names happen to be identical to the names we already
# use so no mapping is necessary.
"event_source": channel,
# This timestamp is reported by the local clock on the device so it may be wildly incorrect.
"time": time,
"context": complete_context,
"page": segment_event.get('page'),
"host": complete_context.get('host', ''),
"agent": '',
"ip": segment_event.get('ip', ''),
"event": segment_event.get('event', {}),
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
tracker.send(event)
return response()
def track_segmentio_event(request):
"""
An endpoint for logging events using segment.io's webhook integration.
segment.io provides a custom integration mechanism that initiates a request to a configurable URL every time an
event is received by their system. This endpoint is designed to receive those requests and convert the events into
standard tracking log entries.
For now we limit the scope of handled events to track and screen events from mobile devices. In the future we could
enable logging of other types of events, however, there is significant overlap with our non-segment.io based event
tracking. Given that segment.io is closed third party solution we are limiting its required usage to just
collecting events from mobile devices for the time being.
Many of the root fields of a standard edX tracking event are read out of the "properties" dictionary provided by the
segment.io event, which is, in turn, provided by the client that emitted the event.
In order for an event to be logged the following preconditions must be met:
* The "key" query string parameter must exactly match the django setting TRACKING_SEGMENTIO_WEBHOOK_SECRET. While
the endpoint is public, we want to limit access to it to the segment.io servers only.
* The value of the "channel" field of the event must be included in the list specified by the django setting
TRACKING_SEGMENTIO_ALLOWED_CHANNELS. This is intended to restrict the set of events to specific channels. For
example: just mobile devices.
* The value of the "action" field of the event must be included in the list specified by the django setting
TRACKING_SEGMENTIO_ALLOWED_ACTIONS. In order to make use of *all* of the features segment.io offers we would have
to implement some sort of persistent storage of information contained in some actions (like identify). For now,
we defer support of those actions and just support a limited set that can be handled without storing information
in external state.
* The value of the standard "userId" field of the event must be an integer that can be used to look up the user
using the primary key of the User model.
* Include an "event_type" field in the properties dictionary that indicates the edX event type. Note this can differ
from the "event" field found in the root of a segment.io event. The "event" field at the root of the structure is
intended to be human readable, the "event_type" field is expected to conform to the standard for naming events
found in the edX data documentation.
Additionally the event can optionally:
* Provide a "context" dictionary in the properties dictionary. This dictionary will be applied to the
existing context on the server overriding any existing keys. This context dictionary should include a "course_id"
field when the event is scoped to a particular course. The value of this field should be a valid course key. The
context may contain other arbitrary data that will be logged with the event, for example: identification
information for the device that emitted the event.
* Provide a "page" parameter in the properties dictionary which indicates the page that was being displayed to the
user or the mobile application screen that was visible to the user at the time the event was emitted.
"""
# Validate the security token. We must use a query string parameter for this since we cannot customize the POST body
# in the segment.io webhook configuration, we can only change the URL that they call, so we force this token to be
# included in the URL and reject any requests that do not include it. This also assumes HTTPS is used to make the
# connection between their server and ours.
expected_secret = getattr(settings, 'TRACKING_SEGMENTIO_WEBHOOK_SECRET',
None)
provided_secret = request.GET.get('key')
if not expected_secret or provided_secret != expected_secret:
return failure_response(ERROR_UNAUTHORIZED, status=401)
# The POST body will contain the JSON encoded event
full_segment_event = request.json
def logged_failure_response(*args, **kwargs):
"""Indicate a failure and log information about the event that will aide debugging efforts"""
failed_response = failure_response(*args, **kwargs)
log.warning('Unable to process event received from segment.io: %s',
json.dumps(full_segment_event))
return failed_response
# Selectively listen to particular channels
channel = full_segment_event.get('channel')
allowed_channels = [
c.lower()
for c in getattr(settings, 'TRACKING_SEGMENTIO_ALLOWED_CHANNELS', [])
]
if not channel or channel.lower() not in allowed_channels:
return response(WARNING_IGNORED_CHANNEL, committed=False)
# Ignore actions that are unsupported
action = full_segment_event.get('action')
allowed_actions = [
a.lower()
for a in getattr(settings, 'TRACKING_SEGMENTIO_ALLOWED_ACTIONS', [])
]
if not action or action.lower() not in allowed_actions:
return response(WARNING_IGNORED_ACTION, committed=False)
# We mostly care about the properties
segment_event = full_segment_event.get('properties', {})
context = {}
# Start with the context provided by segment.io in the "client" field if it exists
segment_context = full_segment_event.get('context')
if segment_context:
context['client'] = segment_context
# Overlay any context provided in the properties
context.update(segment_event.get('context', {}))
user_id = full_segment_event.get('userId')
if not user_id:
return logged_failure_response(ERROR_MISSING_USER_ID)
# userId is assumed to be the primary key of the django User model
try:
user = User.objects.get(pk=user_id)
except User.DoesNotExist:
return logged_failure_response(ERROR_USER_NOT_EXIST)
except ValueError:
return logged_failure_response(ERROR_INVALID_USER_ID)
else:
context['user_id'] = user_id
# course_id is expected to be provided in the context when applicable
course_id = context.get('course_id')
if course_id:
try:
course_key = CourseKey.from_string(course_id)
context['org_id'] = course_key.org
except InvalidKeyError:
log.warning(
'unable to parse course_id "{course_id}" from event: {event}'.
format(
course_id=course_id,
event=json.dumps(full_segment_event),
),
exc_info=True)
if 'timestamp' in full_segment_event:
time = parse_iso8601_timestamp(full_segment_event['timestamp'])
else:
return logged_failure_response(ERROR_MISSING_TIMESTAMP)
if 'receivedAt' in full_segment_event:
context['received_at'] = parse_iso8601_timestamp(
full_segment_event['receivedAt'])
else:
return logged_failure_response(ERROR_MISSING_RECEIVED_AT)
if 'event_type' in segment_event:
event_type = segment_event['event_type']
else:
return logged_failure_response(ERROR_MISSING_EVENT_TYPE)
with eventtracker.get_tracker().context('edx.segmentio', context):
complete_context = eventtracker.get_tracker().resolve_context()
event = {
"username": user.username,
"event_type": event_type,
# Will be either "mobile", "browser" or "server". These names happen to be identical to the names we already
# use so no mapping is necessary.
"event_source": channel,
# This timestamp is reported by the local clock on the device so it may be wildly incorrect.
"time": time,
"context": complete_context,
"page": segment_event.get('page'),
"host": complete_context.get('host', ''),
"agent": '',
"ip": segment_event.get('ip', ''),
"event": segment_event.get('event', {}),
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
tracker.send(event)
return response()
def user_track(request):
"""
Log when POST call to "event" URL is made by a user. Uses request.REQUEST
to allow for GET calls.
GET or POST call should provide "event_type", "event", and "page" arguments.
"""
# EDXFIX: 参照我们以前的track改，确保track日志一致
try: # TODO: Do the same for many of the optional META parameters
username = request.user.username
except:
username = "******"
sessionid = _get_session_id(request, True)
event_type = _get_request_value(request, 'event_type')
if 'event' in request.REQUEST:
try:
page = request.REQUEST['page']
event_obj = request.REQUEST.get('event')
except:
page = ''
event_obj = ''
try:
event_obj = json.loads(event_obj)
except:
pass
else:
page = ''
try:
event_obj = json.loads(request.raw_post_data)
event_type = eventtracker.get_tracker().resolve_context()['path']
except:
event_obj = ''
with eventtracker.get_tracker().context('edx.course.browser', contexts.course_context_from_url(page)):
event = {
"username": username,
"session": sessionid,
"referer": _get_request_header(request, 'HTTP_REFERER'),
"origin_referer": request.session.get('referer'),
"spam": request.COOKIES.get('spam') or request.GET.get('spam'),
"ip": _get_request_header(request, 'REMOTE_ADDR'),
"event_source": "browser",
"event_type": event_type,
"event": event_obj,
"agent": _get_request_header(request, 'HTTP_USER_AGENT'),
"page": page,
"time": datetime.datetime.now(UTC),
"host": _get_request_header(request, 'SERVER_NAME'),
"context": eventtracker.get_tracker().resolve_context(),
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
log_event(event)
return HttpResponse('success')
def server_track(request, event_type, event, page=None):
"""
Log events related to server requests.
Handle the situation where the request may be NULL, as may happen with management commands.
"""
if event_type.startswith("/event_logs") and request.user.is_staff:
return # don't log
event = event or {}
if isinstance(event, dict):
FILED_MAPPER = {
'uid': 'HTTP_UID',
'uuid': 'HTTP_UUID',
'sid': 'HTTP_SID'
}
for key, value in FILED_MAPPER.iteritems():
_filed = _get_request_header(request, value)
if _filed:
event[key] = _filed
else:
event[key] = event.get(key) or ''
if event_type.startswith("/analytic_track"):
event['POST'] = json.loads(request.raw_post_data)
# request.user is an instance of
# django.contrib.auth.models.User if logged in
# django.contrib.auth.models.AnonymousUser if not logged in
if not request:
return
username = request.user.username
sessionid = _get_session_id(request, True)
try:
uid = request.user.id or int(event['uid'])
except:
uid = -1
uuid = event['uuid']
# define output:
event = {
"username": username,
"uid": uid,
"uuid": uuid,
"session": sessionid,
"referer": _get_request_header(request, 'HTTP_REFERER'),
"origin_referer": request.session.get('referer'),
"spam": request.COOKIES.get('spam') or request.GET.get('spam'),
"ip": _get_request_header(request, 'REMOTE_ADDR'),
"event_source": "server",
"event_type": event_type,
"event": event,
"agent": _get_request_header(request, 'HTTP_USER_AGENT'),
"method": request.method,
"page": page,
"time": datetime.datetime.now(UTC),
"host": _get_request_header(request, 'SERVER_NAME'),
"context": eventtracker.get_tracker().resolve_context(),
}
# Some duplicated fields are passed into event-tracking via the context by track.middleware.
# Remove them from the event here since they are captured elsewhere.
shim.remove_shim_context(event)
log_event(event)