def testSubPartitions(self):
test_table_name = tn('pyodps_t_tmp_sub_partitions_table')
root_partition = 'type=test'
sub_partitions = ['s=%s' % i for i in range(3)]
schema = Schema.from_lists([
'id',
], [
'string',
], ['type', 's'], ['string', 'string'])
self.odps.delete_table(test_table_name, if_exists=True)
table = self.odps.create_table(test_table_name, schema)
partitions = [root_partition + ',' + p for p in sub_partitions]
partitions.append('type=test2,s=0')
for partition in partitions:
table.create_partition(partition)
self.assertEqual(
sorted([str(types.PartitionSpec(p)) for p in partitions]),
sorted([str(p.partition_spec) for p in table.partitions]))
self.assertEqual(len(list(table.iterate_partitions(root_partition))),
3)
table.delete_partition(partitions[0])
self.assertEqual(
sorted([str(types.PartitionSpec(p)) for p in partitions[1:]]),
sorted([str(p.partition_spec) for p in table.partitions]))
self.odps.delete_table(test_table_name)
def testPartitions(self):
test_table_name = tn('pyodps_t_tmp_partitions_table')
partitions = ['s=%s' % i for i in range(3)]
schema = Schema.from_lists([
'id',
], [
'string',
], [
's',
], [
'string',
])
self.odps.delete_table(test_table_name, if_exists=True)
table = self.odps.create_table(test_table_name, schema)
for partition in partitions:
table.create_partition(partition)
self.assertEqual(
sorted([str(types.PartitionSpec(p)) for p in partitions]),
sorted([str(p.partition_spec) for p in table.partitions]))
table.get_partition(partitions[0]).drop()
self.assertEqual(
sorted([str(types.PartitionSpec(p)) for p in partitions[1:]]),
sorted([str(p.partition_spec) for p in table.partitions]))
p = next(table.partitions)
self.assertGreater(len(p.columns), 0)
p.reload()
self.assertGreater(len(p.columns), 0)
self.odps.delete_table(test_table_name)
def testTableResource(self):
test_table_name = tn('pyodps_t_tmp_resource_table')
schema = Schema.from_lists(['id', 'name'], ['string', 'string'])
self.odps.delete_table(test_table_name, if_exists=True)
self.odps.create_table(test_table_name, schema)
resource_name = tn('pyodps_t_tmp_table_resource')
try:
self.odps.delete_resource(resource_name)
except errors.NoSuchObject:
pass
res = self.odps.create_resource(resource_name,
'table',
table_name=test_table_name)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertIsNone(res.get_source_table_partition())
self.assertIs(res, self.odps.get_resource(resource_name))
del res.parent[resource_name] # delete from cache
self.assertIsNot(res, self.odps.get_resource(resource_name))
res = self.odps.get_resource(resource_name)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertIsNone(res.get_source_table_partition())
test_table_name = tn('pyodps_t_tmp_resource_table')
test_table_partition = 'pt=test,sec=1'
schema = Schema.from_lists(['id', 'name'], ['string', 'string'],
['pt', 'sec'], ['string', 'bigint'])
self.odps.delete_table(test_table_name, if_exists=True)
table = self.odps.create_table(test_table_name, schema)
table.create_partition(test_table_partition)
resource_name = tn('pyodps_t_tmp_table_resource')
res = res.update(partition=test_table_partition)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertEqual(str(res.get_source_table_partition()),
str(types.PartitionSpec(test_table_partition)))
self.assertIs(res, self.odps.get_resource(resource_name))
test_table_partition = 'pt=test,sec=2'
table.create_partition(test_table_partition)
res = res.update(partition=test_table_partition)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertEqual(str(res.get_source_table_partition()),
str(types.PartitionSpec(test_table_partition)))
self.assertIs(res, self.odps.get_resource(resource_name))
self.odps.delete_resource(resource_name)
self.odps.delete_table(test_table_name)
def testTableResource(self):
secondary_project = self.config.get('test', 'secondary_project')
test_table_name = tn('pyodps_t_tmp_resource_table')
schema = Schema.from_lists(['id', 'name'], ['string', 'string'])
self.odps.delete_table(test_table_name, if_exists=True)
self.odps.create_table(test_table_name, schema)
if secondary_project:
self.odps.delete_table(test_table_name,
if_exists=True,
project=secondary_project)
self.odps.create_table(test_table_name,
schema,
project=secondary_project)
resource_name = tn('pyodps_t_tmp_table_resource')
try:
self.odps.delete_resource(resource_name)
except errors.NoSuchObject:
pass
res = self.odps.create_resource(resource_name,
'table',
table_name=test_table_name)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertEqual(res.table.name, test_table_name)
self.assertIsNone(res.get_source_table_partition())
self.assertIs(res, self.odps.get_resource(resource_name))
with res.open_writer() as writer:
writer.write([0, FILE_CONTENT])
with res.open_reader() as reader:
rec = list(reader)[0]
self.assertEqual(rec[1], FILE_CONTENT)
del res.parent[resource_name] # delete from cache
self.assertIsNot(res, self.odps.get_resource(resource_name))
res = self.odps.get_resource(resource_name)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertIsNone(res.get_source_table_partition())
test_table_partition = 'pt=test,sec=1'
schema = Schema.from_lists(['id', 'name'], ['string', 'string'],
['pt', 'sec'], ['string', 'bigint'])
self.odps.delete_table(test_table_name, if_exists=True)
table = self.odps.create_table(test_table_name, schema)
table.create_partition(test_table_partition)
res = res.update(partition=test_table_partition)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertEqual(res.table.name, test_table_name)
self.assertEqual(str(res.get_source_table_partition()),
str(types.PartitionSpec(test_table_partition)))
self.assertEqual(str(res.partition.spec),
str(types.PartitionSpec(test_table_partition)))
self.assertIs(res, self.odps.get_resource(resource_name))
test_table_partition = 'pt=test,sec=2'
table.create_partition(test_table_partition)
res = res.update(partition=test_table_partition)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertEqual(str(res.get_source_table_partition()),
str(types.PartitionSpec(test_table_partition)))
self.assertIs(res, self.odps.get_resource(resource_name))
test_table_partition = types.PartitionSpec('pt=test,sec=3')
table.create_partition(test_table_partition)
res = res.update(partition=test_table_partition)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertEqual(str(res.get_source_table_partition()),
str(test_table_partition))
self.assertIs(res, self.odps.get_resource(resource_name))
with res.open_writer() as writer:
writer.write([0, FILE_CONTENT])
with res.open_reader() as reader:
rec = list(reader)[0]
self.assertEqual(rec[1], FILE_CONTENT)
if secondary_project:
resource_name2 = tn('pyodps_t_tmp_table_resource2')
try:
self.odps.delete_resource(resource_name2)
except errors.NoSuchObject:
pass
res = self.odps.create_resource(resource_name2,
'table',
project_name=secondary_project,
table_name=test_table_name)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().project.name,
secondary_project)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertEqual(res.table.project.name, secondary_project)
self.assertEqual(res.table.name, test_table_name)
self.assertIsNone(res.get_source_table_partition())
self.assertIs(res, self.odps.get_resource(resource_name2))
del res.parent[resource_name2] # delete from cache
self.assertIsNot(res, self.odps.get_resource(resource_name2))
res = self.odps.get_resource(resource_name2)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().project.name,
secondary_project)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertIsNone(res.get_source_table_partition())
test_table_partition = 'pt=test,sec=1'
res = res.update(project_name=self.odps.project,
partition=test_table_partition)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().project.name,
self.odps.project)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertEqual(str(res.partition.spec),
str(types.PartitionSpec(test_table_partition)))
res = res.update(table_name=secondary_project + '.' +
test_table_name,
partition=None)
self.assertIsInstance(res, TableResource)
self.assertEqual(res.get_source_table().project.name,
secondary_project)
self.assertEqual(res.get_source_table().name, test_table_name)
self.assertIsNone(res.get_source_table_partition())
self.odps.delete_resource(resource_name)
self.odps.delete_table(test_table_name)
if secondary_project:
self.odps.delete_table(test_table_name, project=secondary_project)