Example #1
0
    def requires(self):  # pragma: no cover
        for requirement in super(EnterpriseEnrollmentDataTask,
                                 self).requires():
            yield requirement

        # the process that generates the source table used by this query
        yield (
            ImportAuthUserTask(),
            ImportAuthUserProfileTask(),
            ImportEnterpriseCustomerTask(),
            ImportEnterpriseCustomerUserTask(),
            ImportEnterpriseCourseEnrollmentUserTask(),
            ImportDataSharingConsentTask(),
            ImportUserSocialAuthTask(),
            ImportStudentCourseEnrollmentTask(),
            ImportPersistentCourseGradeTask(),
            CoursePartitionTask(
                date=self.date,
                warehouse_path=self.warehouse_path,
                api_root_url=self.api_root_url,
                api_page_size=self.api_page_size,
            ),
            UserActivityTableTask(warehouse_path=self.warehouse_path,
                                  overwrite_n_days=0,
                                  date=self.date),
        )
Example #2
0
    def test_query_with_date(self):
        dt = '2014-07-01'
        kwargs = {
            'import_date': datetime.datetime.strptime(dt, '%Y-%m-%d').date()
        }
        task = ImportEnterpriseCustomerTask(**kwargs)
        query = task.query()
        expected_query = textwrap.dedent("""
            USE default;
            DROP TABLE IF EXISTS `enterprise_enterprisecustomer`;
            CREATE EXTERNAL TABLE `enterprise_enterprisecustomer` (
                `created` TIMESTAMP,`modified` TIMESTAMP,`uuid` STRING,`name` STRING,`active` BOOLEAN,`site_id` INT,`catalog` INT,`enable_data_sharing_consent` BOOLEAN,`enforce_data_sharing_consent` STRING,`enable_audit_enrollment` BOOLEAN,`enable_audit_data_reporting` BOOLEAN
            )
            PARTITIONED BY (dt STRING)

            LOCATION 's3://foo/bar/enterprise_enterprisecustomer';
            ALTER TABLE `enterprise_enterprisecustomer` ADD PARTITION (dt = '{dt}');
            """.format(dt=dt))
        self.assertEquals(query, expected_query)
    def test_query_with_date(self):
        dt = '2014-07-01'
        kwargs = {'import_date': datetime.datetime.strptime(dt, '%Y-%m-%d').date()}
        task = ImportEnterpriseCustomerTask(**kwargs)
        query = task.query()
        expected_query = textwrap.dedent(
            """
            USE default;
            DROP TABLE IF EXISTS `enterprise_enterprisecustomer`;
            CREATE EXTERNAL TABLE `enterprise_enterprisecustomer` (
                `created` TIMESTAMP,`modified` TIMESTAMP,`uuid` STRING,`name` STRING,`active` BOOLEAN,`site_id` INT,`catalog` INT,`enable_data_sharing_consent` BOOLEAN,`enforce_data_sharing_consent` STRING,`enable_audit_enrollment` BOOLEAN,`enable_audit_data_reporting` BOOLEAN
            )
            PARTITIONED BY (dt STRING)

            LOCATION 's3://foo/bar/enterprise_enterprisecustomer';
            ALTER TABLE `enterprise_enterprisecustomer` ADD PARTITION (dt = '{dt}');
            """.format(dt=dt)
        )
        self.assertEquals(query, expected_query)
Example #4
0
    def requires(self):  # pragma: no cover
        for requirement in super(EnterpriseEnrollmentDataTask, self).requires():
            yield requirement

        # the process that generates the source table used by this query
        yield (
            ImportAuthUserTask(),
            ImportAuthUserProfileTask(),
            ImportEnterpriseCustomerTask(),
            ImportEnterpriseCustomerUserTask(),
            ImportEnterpriseCourseEnrollmentUserTask(),
            ImportDataSharingConsentTask(),
            ImportUserSocialAuthTask(),
            ImportStudentCourseEnrollmentTask(),
            ImportPersistentCourseGradeTask(),
            CoursePartitionTask(
                date=self.date,
                warehouse_path=self.warehouse_path,
                api_root_url=self.api_root_url,
                api_page_size=self.api_page_size,
            ),
            UserActivityTableTask(
                warehouse_path=self.warehouse_path,
                overwrite_n_days=0,
                date=self.date
            ),
            ExternalCourseEnrollmentSummaryPartitionTask(
                date=self.date
            ),
        )

        kwargs = {
            'credentials': self.otto_credentials,
            'database': self.otto_database,
        }
        yield (
            ImportProductCatalog(**kwargs),
            ImportCurrentOrderLineState(**kwargs),
            ImportCurrentOrderDiscountState(**kwargs),
            ImportVoucherTask(**kwargs),
            ImportStockRecordTask(**kwargs),
            ImportCurrentOrderState(**kwargs),
            ImportEcommerceUser(**kwargs),
            ImportConditionalOfferTask(**kwargs),
            ImportBenefitTask(**kwargs),
        )