def requires(self): # pragma: no cover
for requirement in super(EnterpriseEnrollmentDataTask,
self).requires():
yield requirement
# the process that generates the source table used by this query
yield (
ImportAuthUserTask(),
ImportAuthUserProfileTask(),
ImportEnterpriseCustomerTask(),
ImportEnterpriseCustomerUserTask(),
ImportEnterpriseCourseEnrollmentUserTask(),
ImportDataSharingConsentTask(),
ImportUserSocialAuthTask(),
ImportStudentCourseEnrollmentTask(),
ImportPersistentCourseGradeTask(),
CoursePartitionTask(
date=self.date,
warehouse_path=self.warehouse_path,
api_root_url=self.api_root_url,
api_page_size=self.api_page_size,
),
UserActivityTableTask(warehouse_path=self.warehouse_path,
overwrite_n_days=0,
date=self.date),
)
def test_query_with_date(self):
kwargs = {'import_date': datetime.datetime.strptime('2014-07-01', '%Y-%m-%d').date()}
task = ImportPersistentCourseGradeTask(**kwargs)
query = task.query()
expected_query = textwrap.dedent(
"""
USE default;
DROP TABLE IF EXISTS `grades_persistentcoursegrade`;
CREATE EXTERNAL TABLE `grades_persistentcoursegrade` (
`id` INT,`user_id` INT,`course_id` STRING,`course_edited_timestamp` TIMESTAMP,`course_version` STRING,`grading_policy_hash` STRING,`percent_grade` DECIMAL(10,2),`letter_grade` STRING,`passed_timestamp` TIMESTAMP,`created` TIMESTAMP,`modified` TIMESTAMP
)
PARTITIONED BY (dt STRING)
LOCATION 's3://foo/bar/grades_persistentcoursegrade';
ALTER TABLE `grades_persistentcoursegrade` ADD PARTITION (dt = '2014-07-01');
"""
)
self.assertEquals(query, expected_query)
def test_query_with_date(self):
kwargs = {
'import_date':
datetime.datetime.strptime('2014-07-01', '%Y-%m-%d').date()
}
task = ImportPersistentCourseGradeTask(**kwargs)
query = task.query()
expected_query = textwrap.dedent("""
USE default;
DROP TABLE IF EXISTS grades_persistentcoursegrade;
CREATE EXTERNAL TABLE grades_persistentcoursegrade (
id INT,user_id INT,course_id STRING,course_edited_timestamp TIMESTAMP,course_version STRING,grading_policy_hash STRING,percent_grade DECIMAL,letter_grade STRING,passed_timestamp TIMESTAMP,created TIMESTAMP,modified TIMESTAMP
)
PARTITIONED BY (dt STRING)
LOCATION 's3://foo/bar/grades_persistentcoursegrade';
ALTER TABLE grades_persistentcoursegrade ADD PARTITION (dt = '2014-07-01');
""")
self.assertEquals(query, expected_query)
def test_requires(self):
"""The CourseGradeByModeDataTask should require the CourseGradeByModePartitionTask
and the ImportPersistentCourseGradeTask."""
a_date = datetime(2017, 1, 1)
the_warehouse_path = '/tmp/foo'
data_task = CourseGradeByModeDataTask(date=a_date, warehouse_path=the_warehouse_path)
required_tasks = list(data_task.requires())
assert CourseGradeByModePartitionTask(date=a_date, warehouse_path=the_warehouse_path) == required_tasks[0]
assert ImportPersistentCourseGradeTask(import_date=a_date, destination=the_warehouse_path) == required_tasks[1]
def requires(self): # pragma: no cover
for requirement in super(EnterpriseEnrollmentDataTask, self).requires():
yield requirement
# the process that generates the source table used by this query
yield (
ImportAuthUserTask(),
ImportAuthUserProfileTask(),
ImportEnterpriseCustomerTask(),
ImportEnterpriseCustomerUserTask(),
ImportEnterpriseCourseEnrollmentUserTask(),
ImportDataSharingConsentTask(),
ImportUserSocialAuthTask(),
ImportStudentCourseEnrollmentTask(),
ImportPersistentCourseGradeTask(),
CoursePartitionTask(
date=self.date,
warehouse_path=self.warehouse_path,
api_root_url=self.api_root_url,
api_page_size=self.api_page_size,
),
UserActivityTableTask(
warehouse_path=self.warehouse_path,
overwrite_n_days=0,
date=self.date
),
ExternalCourseEnrollmentSummaryPartitionTask(
date=self.date
),
)
kwargs = {
'credentials': self.otto_credentials,
'database': self.otto_database,
}
yield (
ImportProductCatalog(**kwargs),
ImportCurrentOrderLineState(**kwargs),
ImportCurrentOrderDiscountState(**kwargs),
ImportVoucherTask(**kwargs),
ImportStockRecordTask(**kwargs),
ImportCurrentOrderState(**kwargs),
ImportEcommerceUser(**kwargs),
ImportConditionalOfferTask(**kwargs),
ImportBenefitTask(**kwargs),
)