def test_no_overwrite(self): # kwargs = {'overwrite': False} kwargs = {} task = ImportStudentCourseEnrollmentTask(**kwargs) with patch('edx.analytics.tasks.insights.database_imports.HivePartitionTarget') as mock_target: output = mock_target() # Make MagicMock act more like a regular mock, so that flatten() does the right thing. del output.__iter__ del output.__getitem__ output.exists = Mock(return_value=False) self.assertFalse(task.complete()) self.assertTrue(output.exists.called) output.exists = Mock(return_value=True) self.assertTrue(task.complete()) self.assertTrue(output.exists.called)
def test_no_overwrite(self): kwargs = {} task = ImportStudentCourseEnrollmentTask(**kwargs) with patch('edx.analytics.tasks.insights.database_imports.HivePartitionTarget') as mock_target: output = mock_target() # Make MagicMock act more like a regular mock, so that flatten() does the right thing. del output.__iter__ del output.__getitem__ output.exists = Mock(return_value=False) self.assertFalse(task.complete()) self.assertTrue(output.exists.called) output.exists = Mock(return_value=True) self.assertTrue(task.complete()) self.assertTrue(output.exists.called)
def requires(self): # pragma: no cover for requirement in super(EnterpriseEnrollmentDataTask, self).requires(): yield requirement # the process that generates the source table used by this query yield ( ImportAuthUserTask(), ImportAuthUserProfileTask(), ImportEnterpriseCustomerTask(), ImportEnterpriseCustomerUserTask(), ImportEnterpriseCourseEnrollmentUserTask(), ImportDataSharingConsentTask(), ImportUserSocialAuthTask(), ImportStudentCourseEnrollmentTask(), ImportPersistentCourseGradeTask(), CoursePartitionTask( date=self.date, warehouse_path=self.warehouse_path, api_root_url=self.api_root_url, api_page_size=self.api_page_size, ), UserActivityTableTask(warehouse_path=self.warehouse_path, overwrite_n_days=0, date=self.date), )
def test_query_with_date(self): kwargs = {'import_date': datetime.datetime.strptime('2014-07-01', '%Y-%m-%d').date()} task = ImportStudentCourseEnrollmentTask(**kwargs) query = task.query() expected_query = textwrap.dedent( """ USE default; DROP TABLE IF EXISTS `student_courseenrollment`; CREATE EXTERNAL TABLE `student_courseenrollment` ( `id` INT,`user_id` INT,`course_id` STRING,`created` TIMESTAMP,`is_active` BOOLEAN,`mode` STRING ) PARTITIONED BY (dt STRING) LOCATION 's3://foo/bar/student_courseenrollment'; ALTER TABLE `student_courseenrollment` ADD PARTITION (dt = '2014-07-01'); """ ) self.assertEquals(query, expected_query)
def requires(self): kwargs = { 'destination': self.destination, 'num_mappers': self.num_mappers, 'verbose': self.verbose, 'import_date': self.import_date, 'overwrite': self.overwrite, 'credentials': self.otto_credentials, 'database': self.otto_database, } yield ( # Otto User Table ImportEcommerceUser(**kwargs), # Otto Product Tables. ImportProductCatalog(**kwargs), ImportProductCatalogClass(**kwargs), ImportProductCatalogAttributes(**kwargs), ImportProductCatalogAttributeValues(**kwargs), # Otto Current State, Line Item, and Coupon Tables. ImportCurrentOrderState(**kwargs), ImportCurrentOrderLineState(**kwargs), ImportCurrentOrderDiscountState(**kwargs), ImportCouponVoucherIndirectionState(**kwargs), ImportCouponVoucherState(**kwargs), # Otto Refund Tables. ImportCurrentRefundRefundLineState(**kwargs), # Otto Partner Information. ImportEcommercePartner(**kwargs), ) kwargs['credentials'] = self.credentials kwargs['database'] = self.database yield ( # Shopping cart tables. ImportShoppingCartOrder(**kwargs), ImportShoppingCartOrderItem(**kwargs), ImportShoppingCartCertificateItem(**kwargs), ImportShoppingCartPaidCourseRegistration(**kwargs), ImportShoppingCartDonation(**kwargs), ImportShoppingCartCourseRegistrationCodeItem(**kwargs), ImportShoppingCartCoupon(**kwargs), ImportShoppingCartCouponRedemption(**kwargs), # Other LMS tables. ImportAuthUserTask(**kwargs), ImportCourseEntitlementTask(**kwargs), ImportStudentCourseEnrollmentTask(**kwargs), )
def requires(self): yield ( ImportCourseModeTask( import_date=self.import_date ), ImportStudentCourseEnrollmentTask( import_date=self.import_date ), ReconciledOrderTransactionTableTask( import_date=self.import_date, n_reduce_tasks=self.n_reduce_tasks ) )
def requires(self): # pragma: no cover for requirement in super(EnterpriseEnrollmentDataTask, self).requires(): yield requirement # the process that generates the source table used by this query yield ( ImportAuthUserTask(), ImportAuthUserProfileTask(), ImportEnterpriseCustomerTask(), ImportEnterpriseCustomerUserTask(), ImportEnterpriseCourseEnrollmentUserTask(), ImportDataSharingConsentTask(), ImportUserSocialAuthTask(), ImportStudentCourseEnrollmentTask(), ImportPersistentCourseGradeTask(), CoursePartitionTask( date=self.date, warehouse_path=self.warehouse_path, api_root_url=self.api_root_url, api_page_size=self.api_page_size, ), UserActivityTableTask( warehouse_path=self.warehouse_path, overwrite_n_days=0, date=self.date ), ExternalCourseEnrollmentSummaryPartitionTask( date=self.date ), ) kwargs = { 'credentials': self.otto_credentials, 'database': self.otto_database, } yield ( ImportProductCatalog(**kwargs), ImportCurrentOrderLineState(**kwargs), ImportCurrentOrderDiscountState(**kwargs), ImportVoucherTask(**kwargs), ImportStockRecordTask(**kwargs), ImportCurrentOrderState(**kwargs), ImportEcommerceUser(**kwargs), ImportConditionalOfferTask(**kwargs), ImportBenefitTask(**kwargs), )
def requires(self): # Note that import parameters not included are 'destination', 'num_mappers', 'verbose', # and 'date' -- we will use the default values for those. kwargs_for_db_import = { 'overwrite': self.overwrite, } yield ( LastCountryOfUserPartitionTask( mapreduce_engine=self.mapreduce_engine, n_reduce_tasks=self.n_reduce_tasks, source=self.source, pattern=self.pattern, warehouse_path=self.warehouse_path, interval=self.interval, interval_start=self.interval_start, interval_end=self.interval_end, overwrite_n_days=self.overwrite_n_days, geolocation_data=self.geolocation_data, overwrite=self.overwrite, ), ImportStudentCourseEnrollmentTask(**kwargs_for_db_import), )
def test_overwrite(self): kwargs = {'overwrite': True} task = ImportStudentCourseEnrollmentTask(**kwargs) self.assertFalse(task.complete())
def requires_hadoop(self): # Instead of just pointing to the output directory of a dump, let's make sure # there is a dump. # We don't have a way to just dump the Mysql table, so deal with the Hive table # definition as well. yield ImportStudentCourseEnrollmentTask(credentials=self.credentials)
def test_overwrite(self): kwargs = {'overwrite': True} task = ImportStudentCourseEnrollmentTask(**kwargs) self.assertFalse(task.complete())