class ProblemResponseRecord(Record):
"""
Record containing the data for a single user's response to a problem, in a given date range.
If there are multiple questions in a problem, they are spread over separate ProblemResponseRecords.
Note that the course_id field is available from the partition string.
"""
# Data sourced from problem_response tracking logs
course_id = StringField(description='Course containing the problem.')
answer_id = StringField(description='Learner\'s answer ID.')
problem_id = StringField(description='Problem\'s block usage ID.')
problem = StringField(description='Problem display name, at time of answering.')
username = StringField(description='Learner\'s username.')
question = StringField(description='Question\'s display name, at time of answering.')
score = FloatField(description='Score achieved by the learner.')
max_score = FloatField(description='Maximum possible score for the problem.')
correct = BooleanField(nullable=True, description='True if all answers are correct; '
'False if any answers are not correct; '
'None if any answers have unknown correctness.')
answer = DelimitedStringField(description='List of answers the user chose for the question.')
total_attempts = IntegerField(description='Total number of attempts the user has made on the problem.')
first_attempt_date = DateTimeField(description='date/time of the first attempt the user has made on the problem.')
last_attempt_date = DateTimeField(description='date/time of the last attempt the user has made on the problem.')
# Data sourced from course_blocks
location = StringField(description='Problem location in the course, concatenated from Section, Subsection, Unit, '
'and problem display name. Sourced from course_blocks.course_path')
sort_idx = IntegerField(description='Sort index for the problem location. Sourced from course_blocks.sort_idx')
class VideoSegmentSummaryRecord(Record):
"""
Video Segment Summary Information used to populate the video table
"""
pipeline_video_id = StringField(length=255,
nullable=False,
description='A concatenation of the course_id and the HTML encoded ID of the video.'
' Intended to uniquely identify an instance of a video in a particular '
'course. Note that ideally we would use an XBlock usage_id here, but '
'it isn\'t present on the legacy events.')
course_id = StringField(length=255,
nullable=False,
description='Course the video was displayed in. This is an opaque key serialized to '
'a string.')
encoded_module_id = StringField(length=255,
nullable=False,
description='This is the HTML encoded module ID for the video. Ideally this would '
'be an XBlock usage_id, but that data is not present on legacy events.')
duration = IntegerField(description='The video length in seconds. This can be inferred for some videos. We don\'t '
'have reliable metadata for the length of videos in the source data.')
segment_length = IntegerField(description='The length of each segment, in seconds.')
users_at_start = IntegerField(description='The number of users who watched the first segment of the video.')
users_at_end = IntegerField(description='The number of users who watched the end of the video. Note that this is '
'not the number of users who watched the last segment of the video.')
total_viewed_seconds = IntegerField(description='The total number of seconds viewed by all users across all '
'segments.')
class EnterpriseEnrollmentRecord(Record):
"""Summarizes a course's enrollment by gender and date."""
enterprise_id = StringField(length=32, nullable=False, description='')
enterprise_name = StringField(length=255, nullable=False, description='')
lms_user_id = IntegerField(nullable=False, description='')
enterprise_user_id = IntegerField(nullable=False, description='')
course_id = StringField(length=255, nullable=False, description='The course the learner is enrolled in.')
enrollment_created_timestamp = DateTimeField(nullable=False, description='')
user_current_enrollment_mode = StringField(length=32, nullable=False, description='')
consent_granted = BooleanField(description='')
letter_grade = StringField(length=32, description='')
has_passed = BooleanField(description='')
passed_timestamp = DateTimeField(description='')
enterprise_sso_uid = StringField(length=255, description='')
enterprise_site_id = IntegerField(description='')
course_title = StringField(length=255, description='')
course_start = DateTimeField(description='')
course_end = DateTimeField(description='')
course_pacing_type = StringField(length=32, description='')
course_duration_weeks = StringField(length=32, description='')
course_min_effort = IntegerField(description='')
course_max_effort = IntegerField(description='')
user_account_creation_timestamp = DateTimeField(description='')
user_email = StringField(length=255, description='')
user_username = StringField(length=255, description='')
course_key = StringField(length=255, description='')
user_country_code = StringField(length=2, description='')
last_activity_date = DateField(description='')
coupon_name = StringField(length=255, description='')
coupon_code = StringField(length=255, description='')
offer = StringField(length=255, description='')
current_grade = FloatField(description='')
course_price = FloatField(description='')
discount_price = FloatField(description='')
unenrollment_timestamp = DateTimeField(description='')
class CourseSummaryEnrollmentRecord(Record):
"""Recent enrollment summary and metadata for a course."""
course_id = StringField(nullable=False,
length=255,
description='A unique identifier of the course')
catalog_course_title = StringField(nullable=True,
length=255,
normalize_whitespace=True,
description='The name of the course')
catalog_course = StringField(nullable=True,
length=255,
description='Course identifier without run')
start_time = DateTimeField(
nullable=True, description='The date and time that the course begins')
end_time = DateTimeField(
nullable=True, description='The date and time that the course ends')
pacing_type = StringField(nullable=True,
length=255,
description='The type of pacing for this course')
availability = StringField(nullable=True,
length=255,
description='Availability status of the course')
enrollment_mode = StringField(
length=100,
nullable=False,
description='Enrollment mode for the enrollment counts')
count = IntegerField(
nullable=True, description='The count of currently enrolled learners')
count_change_7_days = IntegerField(
nullable=True,
description='Difference in enrollment counts over the past 7 days')
cumulative_count = IntegerField(
nullable=True,
description='The cumulative total of all users ever enrolled')
class LastCountryPerCourseRecord(Record):
"""For a given course, stores aggregates about last country."""
date = DateField(nullable=False, description="Date of course enrollment data.")
course_id = StringField(length=255, nullable=False, description="Course ID for course/last-country pair being counted.")
country_code = StringField(length=10, nullable=True, description="Code for country in course/last-country pair being counted.")
count = IntegerField(nullable=False, description="Number enrolled in course whose current last country code matches.")
cumulative_count = IntegerField(nullable=False, description="Number ever enrolled in course whose current last-country code matches.")
class VideoTimelineRecord(Record):
"""
Video Segment Information used to populate the video_timeline table
"""
pipeline_video_id = StringField(
length=255,
nullable=False,
description=
'A concatenation of the course_id and the HTML encoded ID of the video.'
' Intended to uniquely identify an instance of a video in a particular '
'course. Note that ideally we would use an XBlock usage_id here, but '
'it isn\'t present on the legacy events.')
segment = IntegerField(
description=
'An integer representing the rank of the segment within the video. 0 is the '
'first segment, 1 is the second etc. Note that this does not specify the length '
'of the segment.')
num_users = IntegerField(
description=
'The number of unique users who watched any part of this segment of the '
'video.')
num_views = IntegerField(
description=
'The total number of times any part of the segment was viewed, regardless of '
'who was watching it.')
class TagsDistributionRecord(Record):
"""Represents a count of total/correct submissions for the particular bunch (problem_id, tag_key, tag_value)."""
course_id = StringField(length=255, nullable=False, description='Course id')
org_id = StringField(length=255, nullable=False, description='Org id')
module_id = StringField(length=255, nullable=False, description='Problem id')
tag_name = StringField(length=255, nullable=False, description='Tag key')
tag_value = StringField(length=255, nullable=False, description='Tag value')
total_submissions = IntegerField(nullable=False, description='Number of total submissions')
correct_submissions = IntegerField(nullable=False, description='Number of correct submissions')
class EnterpriseUserRecord(Record):
"""Summarizes an enterprise user"""
enterprise_id = StringField(length=32, nullable=False, description='')
lms_user_id = IntegerField(nullable=False, description='')
enterprise_user_id = IntegerField(nullable=False, description='')
enterprise_sso_uid = StringField(length=255, description='')
user_account_creation_timestamp = DateTimeField(description='')
user_email = StringField(length=255, description='')
user_username = StringField(length=255, description='')
user_country_code = StringField(length=2, description='')
last_activity_date = DateField(description='')
class GradesPersistentCourseGradeRecord(Record):
id = IntegerField()
user_id = IntegerField()
course_id = StringField()
course_edited_timestamp = DateTimeField()
course_version = StringField()
grading_policy_hash = StringField()
percent_grade = FloatField()
letter_grade = StringField()
passed_timestamp = DateTimeField()
created = DateTimeField()
modified = DateTimeField()
class CourseActivityRecord(Record):
"""Represents count of users performing each category of activity each ISO week."""
course_id = StringField(length=255, nullable=False, description='The course the learner is enrolled in.')
interval_start = DateTimeField(nullable=False, description='Start time of ISO week.')
interval_end = DateTimeField(nullable=False, description='End time of ISO week.')
label = StringField(length=255, nullable=False, description='The name of activity user performed in the interval.')
count = IntegerField(description='Total count of activities performed between the interval.')
class CourseBlockRecord(Record):
"""
Represents a course block as fetched from the edX Course Blocks REST API, augmented with details about its
position in the course.
"""
block_id = StringField(length=564, nullable=False, description='Block identifier.')
course_id = StringField(length=255, nullable=False, description='Identifier for the course containing the block.')
block_type = StringField(length=255, nullable=False,
description='Block type, e.g. `video`, `chapter`, `problem`, `vectordraw`.')
display_name = StringField(length=255, nullable=False, truncate=True, normalize_whitespace=True,
description='User-facing title of the block. Will be truncated to 255 characters.')
is_root = BooleanField(default=False, nullable=False,
description='True if the block is the course\'s root node.')
is_orphan = BooleanField(default=False, nullable=False,
description='True if the block has no parent nodes, but is not a root node.')
is_dag = BooleanField(default=False, nullable=False,
description='True if the block has more than one parent, making the course a Directed '
'Acyclic Graph. If True, parent_block_id, course_path, and sort_idx will be set '
'to the first place the block is found when traversing the course blocks tree.')
parent_block_id = StringField(length=255, nullable=True,
description='Block identifier for the block\'s parent.')
course_path = StringField(nullable=True, normalize_whitespace=True,
description='Concatenated string of parent block display_name values, from '
'the root node to the parent_block_id.')
sort_idx = IntegerField(nullable=True,
description='Number indicating the position that this block holds in a course-outline '
'sorted list of blocks. See `CourseBlocksApiDataTask.sort_orphan_blocks_up`.')
class VideoSegmentDetailRecord(Record):
"""
Video Segment Usage Detail
"""
pipeline_video_id = StringField(length=255,
nullable=False,
description='A concatenation of the course_id and the HTML encoded ID of the video.'
' Intended to uniquely identify an instance of a video in a particular '
'course. Note that ideally we would use an XBlock usage_id here, but '
'it isn\'t present on the legacy events.')
course_id = StringField(length=255,
nullable=False,
description='Course the video was displayed in. This is an opaque key serialized to '
'a string.')
encoded_module_id = StringField(length=255,
nullable=False,
description='This is the HTML encoded module ID for the video. Ideally this would '
'be an XBlock usage_id, but that data is not present on legacy events.')
duration = IntegerField(description='The video length in seconds. This can be inferred for some videos. We don\'t '
'have reliable metadata for the length of videos in the source data.')
segment_length = IntegerField(description='The length of each segment, in seconds.')
users_at_start = IntegerField(description='The number of users who watched the first segment of the video.')
users_at_end = IntegerField(description='The number of users who watched the end of the video. Note that this is '
'not the number of users who watched the last segment of the video.')
segment = IntegerField(description='An integer representing the rank of the segment within the video. 0 is the '
'first segment, 1 is the second etc. Note that this does not specify the length '
'of the segment.')
num_users = IntegerField(description='The number of unique users who watched any part of this segment of the '
'video.')
num_views = IntegerField(description='The total number of times any part of the segment was viewed, regardless of '
'who was watching it.')
class CourseSeatRecord(Record):
"""Represents a course seat within course run."""
course_id = StringField(nullable=False, length=255)
course_seat_type = StringField(nullable=False, length=255)
course_seat_price = FloatField(nullable=False)
course_seat_currency = StringField(nullable=False, length=255)
course_seat_upgrade_deadline = DateTimeField(nullable=True)
course_seat_credit_provider = StringField(nullable=True, length=255)
course_seat_credit_hours = IntegerField(nullable=True)
class LastIpAddressRecord(Record):
"""
Store information about last IP address observed for a given user in a given course.
Values are not written to a database, so string lengths are not specified.
"""
timestamp = StringField(description='Timestamp of last event by user in a course.')
ip_address = StringField(description='IP address recorded on last event by user in a course.')
user_id = IntegerField(description='User ID recorded on last event by user in a course.')
course_id = StringField(description='Course ID recorded on last event by user in a course.')
class CourseRecord(Record):
"""Represents a course."""
course_id = StringField(nullable=False, length=255)
catalog_course = StringField(nullable=False, length=255)
catalog_course_title = StringField(nullable=True,
length=255,
normalize_whitespace=True)
start_time = DateTimeField(nullable=True)
end_time = DateTimeField(nullable=True)
enrollment_start_time = DateTimeField(nullable=True)
enrollment_end_time = DateTimeField(nullable=True)
content_language = StringField(nullable=True, length=50)
pacing_type = StringField(nullable=True, length=255)
level_type = StringField(nullable=True, length=255)
availability = StringField(nullable=True, length=255)
org_id = StringField(nullable=False, length=255)
partner_short_code = StringField(nullable=True, length=8)
marketing_url = StringField(nullable=True, length=1024)
min_effort = IntegerField(nullable=True)
max_effort = IntegerField(nullable=True)
class ProgramCourseRecord(Record):
"""Represents a course run within a program."""
program_id = StringField(nullable=False, length=36)
program_type = StringField(nullable=False, length=32)
program_title = StringField(nullable=True, length=255, normalize_whitespace=True)
catalog_course = StringField(nullable=False, length=255)
catalog_course_title = StringField(nullable=True, length=255, normalize_whitespace=True)
course_id = StringField(nullable=False, length=255)
org_id = StringField(nullable=True, length=255)
partner_short_code = StringField(nullable=True, length=8)
program_slot_number = IntegerField(nullable=True)
class AccountRecord(Record):
"""Represents a google analytics account."""
account_id = IntegerField(description='Google Analytics Account ID',
nullable=False)
account_name = StringField(description='Google Analytics Account Name',
nullable=False,
length=200)
created = DateTimeField(description='Time the account was created.',
nullable=False)
updated = DateTimeField(description='Time the account was last modified.',
nullable=False)
class ProfileRecord(Record):
"""Represents a Google Analytics View (profile)."""
property_id = StringField(
description='Web property ID to which this view (profile) belongs.',
nullable=False,
length=20)
profile_id = IntegerField(description='View (Profile) ID.', nullable=False)
profile_name = StringField(description='Name of this view (profile).',
nullable=False,
length=200)
profile_type = StringField(description='View (Profile) type. WEB or APP.',
nullable=False,
length=10)
created = DateTimeField(
description='Time this view (profile) was created.', nullable=False)
updated = DateTimeField(
description='Time this view (profile) was last modified.',
nullable=False)
class PropertyRecord(Record):
"""Represents a Google Analytics Web Property."""
account_id = IntegerField(
description='Account ID to which this web property belongs.',
nullable=False)
property_id = StringField(
description='Web property ID of the form UA-XXXXX-YY.',
nullable=False,
length=20)
property_name = StringField(description='Name of this web property.',
nullable=False,
length=200)
website_url = StringField(description='Website url for this web property.',
nullable=True,
length=255)
created = DateTimeField(description='Time this web property was created.',
nullable=False)
updated = DateTimeField(
description='Time this web property was last modified.',
nullable=False)
class EnrollmentSummaryRecord(Record):
"""Summarizes a user's enrollment history for a particular course."""
course_id = StringField(length=255,
nullable=False,
description='Course the learner enrolled in.')
user_id = IntegerField(nullable=False,
description='The user\'s numeric identifier.')
current_enrollment_mode = StringField(
length=100,
nullable=False,
description='The last mode seen on an activation or mode change event.'
)
current_enrollment_is_active = BooleanField(
nullable=False,
description=
'True if the user is currently enrolled as of the end of the interval.'
)
first_enrollment_mode = StringField(
length=100,
nullable=True,
description='The mode the user first enrolled with.')
first_enrollment_time = DateTimeField(
nullable=True, description='The time of the user\'s first enrollment.')
last_unenrollment_time = DateTimeField(
nullable=True,
description='The time of the user\'s last unenrollment.')
first_verified_enrollment_time = DateTimeField(
nullable=True,
description='The time the user first switched to the verified track.')
first_credit_enrollment_time = DateTimeField(
nullable=True,
description='The time the user first switched to the credit track.')
end_time = DateTimeField(
nullable=False,
description='The end of the interval that was analyzed.')
class SampleStruct(Record):
"""A record with a variety of field types"""
name = StringField()
index = IntegerField()
date = DateField()
def test_validate_success(self, value):
test_record = IntegerField()
self.assertEqual(len(test_record.validate(value)), 0)
def test_validate_success(self, value):
test_record = IntegerField()
self.assertEqual(len(test_record.validate(value)), 0)
class IdForAcountPermission(object):
account_id = IntegerField(description='Account ID.', nullable=False)
class IdForProfilePermission(object):
profile_id = IntegerField(description='View (profile) ID.', nullable=False)
def test_sql_type(self):
self.assertEqual(IntegerField().sql_type, 'INT')
def test_validate_error(self, value):
test_record = IntegerField()
self.assertEqual(len(test_record.validate(value)), 1)
def test_hive_type(self):
self.assertEqual(IntegerField().hive_type, 'INT')
def test_validate_error(self, value):
test_record = IntegerField()
self.assertEqual(len(test_record.validate(value)), 1)
class LastCountryOfUserRecord(Record):
"""For a given user_id, stores information about last country."""
country_name = StringField(length=255, description="Name of last country.")
country_code = StringField(length=10, description="Code for last country.")
user_id = IntegerField(description="User ID of user with country information.")
class SampleElasticSearchStruct(Record):
"""A record with a variety of field types to illustrate all elasticsearch properties"""
name = StringField()
index = IntegerField(analyzed=True)
date = DateField()
dateTime = DateTimeField()
class DifferentFieldType(Record):
"""A record with exactly the same fields as SingleFieldRecord but with a differently typed field"""
name = IntegerField()