コード例 #1
0
    def testSubPartitions(self):
        test_table_name = tn('pyodps_t_tmp_sub_partitions_table')
        root_partition = 'type=test'
        sub_partitions = ['s=%s' % i for i in range(3)]
        schema = Schema.from_lists([
            'id',
        ], [
            'string',
        ], ['type', 's'], ['string', 'string'])

        self.odps.delete_table(test_table_name, if_exists=True)
        table = self.odps.create_table(test_table_name, schema)
        partitions = [root_partition + ',' + p for p in sub_partitions]
        partitions.append('type=test2,s=0')
        for partition in partitions:
            table.create_partition(partition)

        self.assertEqual(
            sorted([str(types.PartitionSpec(p)) for p in partitions]),
            sorted([str(p.partition_spec) for p in table.partitions]))

        self.assertEqual(len(list(table.iterate_partitions(root_partition))),
                         3)

        table.delete_partition(partitions[0])
        self.assertEqual(
            sorted([str(types.PartitionSpec(p)) for p in partitions[1:]]),
            sorted([str(p.partition_spec) for p in table.partitions]))

        self.odps.delete_table(test_table_name)
コード例 #2
0
    def testPartitions(self):
        test_table_name = tn('pyodps_t_tmp_partitions_table')
        partitions = ['s=%s' % i for i in range(3)]
        schema = Schema.from_lists([
            'id',
        ], [
            'string',
        ], [
            's',
        ], [
            'string',
        ])

        self.odps.delete_table(test_table_name, if_exists=True)
        table = self.odps.create_table(test_table_name, schema)
        for partition in partitions:
            table.create_partition(partition)

        self.assertEqual(
            sorted([str(types.PartitionSpec(p)) for p in partitions]),
            sorted([str(p.partition_spec) for p in table.partitions]))

        table.get_partition(partitions[0]).drop()
        self.assertEqual(
            sorted([str(types.PartitionSpec(p)) for p in partitions[1:]]),
            sorted([str(p.partition_spec) for p in table.partitions]))

        p = next(table.partitions)
        self.assertGreater(len(p.columns), 0)
        p.reload()
        self.assertGreater(len(p.columns), 0)

        self.odps.delete_table(test_table_name)
コード例 #3
0
    def testTableResource(self):
        test_table_name = tn('pyodps_t_tmp_resource_table')
        schema = Schema.from_lists(['id', 'name'], ['string', 'string'])
        self.odps.delete_table(test_table_name, if_exists=True)
        self.odps.create_table(test_table_name, schema)

        resource_name = tn('pyodps_t_tmp_table_resource')
        try:
            self.odps.delete_resource(resource_name)
        except errors.NoSuchObject:
            pass
        res = self.odps.create_resource(resource_name,
                                        'table',
                                        table_name=test_table_name)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertIsNone(res.get_source_table_partition())
        self.assertIs(res, self.odps.get_resource(resource_name))

        del res.parent[resource_name]  # delete from cache

        self.assertIsNot(res, self.odps.get_resource(resource_name))
        res = self.odps.get_resource(resource_name)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertIsNone(res.get_source_table_partition())

        test_table_name = tn('pyodps_t_tmp_resource_table')
        test_table_partition = 'pt=test,sec=1'
        schema = Schema.from_lists(['id', 'name'], ['string', 'string'],
                                   ['pt', 'sec'], ['string', 'bigint'])
        self.odps.delete_table(test_table_name, if_exists=True)
        table = self.odps.create_table(test_table_name, schema)
        table.create_partition(test_table_partition)

        resource_name = tn('pyodps_t_tmp_table_resource')
        res = res.update(partition=test_table_partition)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertEqual(str(res.get_source_table_partition()),
                         str(types.PartitionSpec(test_table_partition)))
        self.assertIs(res, self.odps.get_resource(resource_name))

        test_table_partition = 'pt=test,sec=2'
        table.create_partition(test_table_partition)
        res = res.update(partition=test_table_partition)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertEqual(str(res.get_source_table_partition()),
                         str(types.PartitionSpec(test_table_partition)))
        self.assertIs(res, self.odps.get_resource(resource_name))

        self.odps.delete_resource(resource_name)
        self.odps.delete_table(test_table_name)
コード例 #4
0
    def testTableResource(self):
        secondary_project = self.config.get('test', 'secondary_project')

        test_table_name = tn('pyodps_t_tmp_resource_table')
        schema = Schema.from_lists(['id', 'name'], ['string', 'string'])
        self.odps.delete_table(test_table_name, if_exists=True)
        self.odps.create_table(test_table_name, schema)
        if secondary_project:
            self.odps.delete_table(test_table_name,
                                   if_exists=True,
                                   project=secondary_project)
            self.odps.create_table(test_table_name,
                                   schema,
                                   project=secondary_project)

        resource_name = tn('pyodps_t_tmp_table_resource')
        try:
            self.odps.delete_resource(resource_name)
        except errors.NoSuchObject:
            pass
        res = self.odps.create_resource(resource_name,
                                        'table',
                                        table_name=test_table_name)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertEqual(res.table.name, test_table_name)
        self.assertIsNone(res.get_source_table_partition())
        self.assertIs(res, self.odps.get_resource(resource_name))

        with res.open_writer() as writer:
            writer.write([0, FILE_CONTENT])
        with res.open_reader() as reader:
            rec = list(reader)[0]
            self.assertEqual(rec[1], FILE_CONTENT)

        del res.parent[resource_name]  # delete from cache

        self.assertIsNot(res, self.odps.get_resource(resource_name))
        res = self.odps.get_resource(resource_name)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertIsNone(res.get_source_table_partition())

        test_table_partition = 'pt=test,sec=1'
        schema = Schema.from_lists(['id', 'name'], ['string', 'string'],
                                   ['pt', 'sec'], ['string', 'bigint'])
        self.odps.delete_table(test_table_name, if_exists=True)
        table = self.odps.create_table(test_table_name, schema)
        table.create_partition(test_table_partition)

        res = res.update(partition=test_table_partition)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertEqual(res.table.name, test_table_name)
        self.assertEqual(str(res.get_source_table_partition()),
                         str(types.PartitionSpec(test_table_partition)))
        self.assertEqual(str(res.partition.spec),
                         str(types.PartitionSpec(test_table_partition)))
        self.assertIs(res, self.odps.get_resource(resource_name))

        test_table_partition = 'pt=test,sec=2'
        table.create_partition(test_table_partition)
        res = res.update(partition=test_table_partition)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertEqual(str(res.get_source_table_partition()),
                         str(types.PartitionSpec(test_table_partition)))
        self.assertIs(res, self.odps.get_resource(resource_name))

        test_table_partition = types.PartitionSpec('pt=test,sec=3')
        table.create_partition(test_table_partition)
        res = res.update(partition=test_table_partition)
        self.assertIsInstance(res, TableResource)
        self.assertEqual(res.get_source_table().name, test_table_name)
        self.assertEqual(str(res.get_source_table_partition()),
                         str(test_table_partition))
        self.assertIs(res, self.odps.get_resource(resource_name))

        with res.open_writer() as writer:
            writer.write([0, FILE_CONTENT])
        with res.open_reader() as reader:
            rec = list(reader)[0]
            self.assertEqual(rec[1], FILE_CONTENT)

        if secondary_project:
            resource_name2 = tn('pyodps_t_tmp_table_resource2')
            try:
                self.odps.delete_resource(resource_name2)
            except errors.NoSuchObject:
                pass
            res = self.odps.create_resource(resource_name2,
                                            'table',
                                            project_name=secondary_project,
                                            table_name=test_table_name)
            self.assertIsInstance(res, TableResource)
            self.assertEqual(res.get_source_table().project.name,
                             secondary_project)
            self.assertEqual(res.get_source_table().name, test_table_name)
            self.assertEqual(res.table.project.name, secondary_project)
            self.assertEqual(res.table.name, test_table_name)
            self.assertIsNone(res.get_source_table_partition())
            self.assertIs(res, self.odps.get_resource(resource_name2))

            del res.parent[resource_name2]  # delete from cache

            self.assertIsNot(res, self.odps.get_resource(resource_name2))
            res = self.odps.get_resource(resource_name2)
            self.assertIsInstance(res, TableResource)
            self.assertEqual(res.get_source_table().project.name,
                             secondary_project)
            self.assertEqual(res.get_source_table().name, test_table_name)
            self.assertIsNone(res.get_source_table_partition())

            test_table_partition = 'pt=test,sec=1'
            res = res.update(project_name=self.odps.project,
                             partition=test_table_partition)
            self.assertIsInstance(res, TableResource)
            self.assertEqual(res.get_source_table().project.name,
                             self.odps.project)
            self.assertEqual(res.get_source_table().name, test_table_name)
            self.assertEqual(str(res.partition.spec),
                             str(types.PartitionSpec(test_table_partition)))

            res = res.update(table_name=secondary_project + '.' +
                             test_table_name,
                             partition=None)
            self.assertIsInstance(res, TableResource)
            self.assertEqual(res.get_source_table().project.name,
                             secondary_project)
            self.assertEqual(res.get_source_table().name, test_table_name)
            self.assertIsNone(res.get_source_table_partition())

        self.odps.delete_resource(resource_name)
        self.odps.delete_table(test_table_name)
        if secondary_project:
            self.odps.delete_table(test_table_name, project=secondary_project)