class LastCountryPerCourseRecord(Record):
"""For a given course, stores aggregates about last country."""
date = DateField(nullable=False, description="Date of course enrollment data.")
course_id = StringField(length=255, nullable=False, description="Course ID for course/last-country pair being counted.")
country_code = StringField(length=10, nullable=True, description="Code for country in course/last-country pair being counted.")
count = IntegerField(nullable=False, description="Number enrolled in course whose current last country code matches.")
cumulative_count = IntegerField(nullable=False, description="Number ever enrolled in course whose current last-country code matches.")
class EnterpriseEnrollmentRecord(Record):
"""Summarizes a course's enrollment by gender and date."""
enterprise_id = StringField(length=32, nullable=False, description='')
enterprise_name = StringField(length=255, nullable=False, description='')
lms_user_id = IntegerField(nullable=False, description='')
enterprise_user_id = IntegerField(nullable=False, description='')
course_id = StringField(length=255, nullable=False, description='The course the learner is enrolled in.')
enrollment_created_timestamp = DateTimeField(nullable=False, description='')
user_current_enrollment_mode = StringField(length=32, nullable=False, description='')
consent_granted = BooleanField(description='')
letter_grade = StringField(length=32, description='')
has_passed = BooleanField(description='')
passed_timestamp = DateTimeField(description='')
enterprise_sso_uid = StringField(length=255, description='')
enterprise_site_id = IntegerField(description='')
course_title = StringField(length=255, description='')
course_start = DateTimeField(description='')
course_end = DateTimeField(description='')
course_pacing_type = StringField(length=32, description='')
course_duration_weeks = StringField(length=32, description='')
course_min_effort = IntegerField(description='')
course_max_effort = IntegerField(description='')
user_account_creation_timestamp = DateTimeField(description='')
user_email = StringField(length=255, description='')
user_username = StringField(length=255, description='')
course_key = StringField(length=255, description='')
user_country_code = StringField(length=2, description='')
last_activity_date = DateField(description='')
coupon_name = StringField(length=255, description='')
coupon_code = StringField(length=255, description='')
offer = StringField(length=255, description='')
current_grade = FloatField(description='')
course_price = FloatField(description='')
discount_price = FloatField(description='')
unenrollment_timestamp = DateTimeField(description='')
class CourseSubjectRecord(Record):
"""Represents course subject categorizations for a course."""
course_id = StringField(nullable=False, length=200)
date = DateField(nullable=True)
subject_uri = StringField(nullable=True, length=200)
subject_title = StringField(nullable=True, length=200)
subject_language = StringField(nullable=True, length=200)
class EnterpriseUserRecord(Record):
"""Summarizes an enterprise user"""
enterprise_id = StringField(length=32, nullable=False, description='')
lms_user_id = IntegerField(nullable=False, description='')
enterprise_user_id = IntegerField(nullable=False, description='')
enterprise_sso_uid = StringField(length=255, description='')
user_account_creation_timestamp = DateTimeField(description='')
user_email = StringField(length=255, description='')
user_username = StringField(length=255, description='')
user_country_code = StringField(length=2, description='')
last_activity_date = DateField(description='')
def run(self):
"""
Read Report file and reformat output
Skip initial 3 lines that provide information about file source and header
4th line should contain header information. We are only interested in "Section Body Data" in docs
In processed file we are saving "Section Body Data" only.
"""
with self.input().open('r') as input_file:
file_read = input_file.readlines()[3:]
reader = csv.DictReader(file_read, delimiter=',')
date_time_field = DateTimeField()
with self.output().open('w') as output_file:
for row in filter(self.filtercsv_row, reader):
if row['Response due date'] == '':
response_date = None
else:
response_date = date_time_field.deserialize_from_string(row['Response due date'])
record = PayPalCaseReportRecord(
case_type=row['Case type'],
case_id=row['Case ID'],
original_transaction_id=row['Original transaction ID'],
transaction_date=date_time_field.deserialize_from_string(row['Transaction date']),
transaction_invoice_id=row['Transaction invoice ID'],
card_type=row['Card Type'],
case_reason=row['Case reason'],
claimant_name=row['Claimant name'],
claimant_email_address=row['Claimant email address'],
case_filing_date=date_time_field.deserialize_from_string(row['Case filing date']),
case_status=row['Case status'],
response_due_date=response_date,
disputed_amount=self.amount_to_decimal(row['Disputed amount']),
disputed_currency=row['Disputed currency'],
disputed_transaction_id=row['Disputed transaction ID'],
money_movement=row['Money movement'],
settlement_type=row['Settlement type'],
seller_protection=row['Seller protection'],
seller_protection_payout_amount=self.amount_to_decimal(row['Seller protection payout amount']),
seller_protection_currency=row['Seller protection currency'],
payment_tracking_id=row['Payment Tracking ID'],
buyer_comments=row['Buyer comments'],
store_id=row['Store ID'],
chargeback_reason_code=row['Chargeback Reason Code'],
outcome=row['Outcome'],
report_generation_date=DateField().deserialize_from_string(self.run_date.strftime('%Y-%m-%d'))
)
output_file.write(record.to_separated_values())
output_file.write('\n')
class StudentEngagementIntervalTypeRecord(Record):
"""
Student Engagement information used to populate student_engagement_{interval_type} tables.
"""
end_date = DateField(
description='End date of the interval being analyzed.')
course_id = StringField(nullable=False,
length=255,
description='Identifier of course run.')
username = StringField(
nullable=False,
length=255,
description=
'The username of the user who was logged in when the event was emitted.'
)
days_active = IntegerField(
description='Count of days user has been active during the interval.')
problems_attempted = IntegerField(
description='Count of unique problems attempted.')
problem_attempts = IntegerField(
description='Total count of problem attempts.')
problems_correct = IntegerField(
description='Count of unique problems that were answered correctly.')
videos_played = IntegerField(description='Count of unique videos played.')
forum_posts = IntegerField(description='Count of discussion posts.')
forum_responses = IntegerField(
description='Count of discussion responses created by user.')
forum_comments = IntegerField(
description='Count of discussion comments by user.')
forum_upvotes_given = IntegerField(
description='Total upvotes given by user on discussion posts.')
forum_downvotes_given = IntegerField(
description='Total downvotes given by user on discussion posts.')
forum_upvotes_received = IntegerField(
description='Total upvotes received by user on discussion posts.')
forum_downvotes_received = IntegerField(
description='Total downvotes received by user on discussion posts.')
textbook_pages_viewed = IntegerField(
description='Total textbook pages viewed by user.')
last_subsection_viewed = StringField(
nullable=False,
length=1000,
description=
'Page URL which was last visited by user during the interval.')
def test_serialize_to_string(self):
self.assertEqual(
DateField().serialize_to_string(datetime.date(2015, 11, 1)),
'2015-11-01')
def test_hive_type(self):
self.assertEqual(DateField().hive_type, 'STRING')
def test_sql_type(self):
self.assertEqual(DateField().sql_type, 'DATE')
def test_validate_error(self, value):
test_record = DateField()
self.assertEqual(len(test_record.validate(value)), 1)
def test_validate_success(self, value):
test_record = DateField()
self.assertEqual(len(test_record.validate(value)), 0)
class SampleElasticSearchStruct(Record):
"""A record with a variety of field types to illustrate all elasticsearch properties"""
name = StringField()
index = IntegerField(analyzed=True)
date = DateField()
dateTime = DateTimeField()
class SampleStruct(Record):
"""A record with a variety of field types"""
name = StringField()
index = IntegerField()
date = DateField()
def test_validate_error(self, value):
test_record = DateField()
self.assertEqual(len(test_record.validate(value)), 1)
def test_validate_success(self, value):
test_record = DateField()
self.assertEqual(len(test_record.validate(value)), 0)
class PayPalCaseReportRecord(Record):
"""Record in PayPal Case Report """
case_type = StringField(
length=50, nullable=True,
description='Type of case made against the transaction for e.g. Chargeback, Dispute, Claim etc'
)
case_id = StringField(
length=18, nullable=True,
description='PayPal generated unique ID for Case'
)
original_transaction_id = StringField(
length=255, nullable=True,
description='PayPal generated ID of transaction against which Case was filed'
)
transaction_date = DateTimeField(
nullable=True,
description='Completion date of the transaction'
)
transaction_invoice_id = StringField(
length=255, nullable=True,
description='Invoice ID provided with transaction'
)
card_type = StringField(
length=255, nullable=True,
description='Credit Card type used for the transaction'
)
case_reason = StringField(
length=255, nullable=True,
description='Systematic reason for the case like Inquiry by PayPal, Item not received, Not as described etc'
)
claimant_name = StringField(
length=128, nullable=True,
description='Name of the claimant'
)
claimant_email_address = StringField(
length=128, nullable=True,
description='PayPal email address of the buyer'
)
case_filing_date = DateTimeField(
nullable=True,
description='Date that the case was originally filed with PayPal'
)
case_status = StringField(
length=255, nullable=True,
description='State or the status of case'
)
response_due_date = DateTimeField(
nullable=True,
description='Date by which filed case should be responded'
)
disputed_amount = IntegerField(
nullable=True,
description='Amount being disputed by the buyer in the original transaction'
)
disputed_currency = StringField(
length=255, nullable=True,
description='Currency of the disputed amount'
)
disputed_transaction_id = StringField(
length=255, nullable=True,
description='Transaction ID generated at the time of the money movement event'
)
money_movement = StringField(
length=255, nullable=True,
description='PayPal amount status like Credit, Debit, On temporary hold etc'
)
settlement_type = StringField(
length=255, nullable=True,
description='Mode to return money to buyer'
)
seller_protection = StringField(
length=255, nullable=True,
description='Specifies the Seller Protection status'
)
seller_protection_payout_amount = IntegerField(
nullable=True,
description='Amount PayPal paid on the behalf of the seller to the buyer as a result of the Seller Protection coverage'
)
seller_protection_currency = StringField(
length=255, nullable=True,
description='Seller protection currency'
)
payment_tracking_id = StringField(
length=255, nullable=True,
description='Unique ID to obtain information about payment or refund'
)
buyer_comments = StringField(
length=500, nullable=True,
description='Comments by buyer'
)
store_id = StringField(
length=255, nullable=True,
description='Merchant identifier of the store where the purchase occurred'
)
chargeback_reason_code = StringField(
length=32, nullable=True,
description='Unique identifier to distinguish chargeback nature / reason'
)
outcome = StringField(
length=3000, nullable=True,
description='Outcome'
)
report_generation_date = DateField(
length=10, nullable=False,
description='Report file generation date'
)