def testListInstancesInPage(self):
test_table = tn('pyodps_t_tmp_list_instances_in_page')
delay_udf = textwrap.dedent("""
from odps.udf import annotate
import sys
import time
@annotate("bigint->bigint")
class Delayer(object):
def evaluate(self, arg0):
print('Start Logging')
sys.stdout.flush()
time.sleep(45)
print('End Logging')
sys.stdout.flush()
return arg0
""")
resource_name = tn('test_delayer_function_resource')
function_name = tn('test_delayer_function')
if self.odps.exist_resource(resource_name + '.py'):
self.odps.delete_resource(resource_name + '.py')
res = self.odps.create_resource(resource_name + '.py', 'py', file_obj=delay_udf)
if self.odps.exist_function(function_name):
self.odps.delete_function(function_name)
fun = self.odps.create_function(function_name, class_type=resource_name + '.Delayer', resources=[res, ])
data = [[random.randint(0, 1000)] for _ in compat.irange(100)]
self.odps.delete_table(test_table, if_exists=True)
t = self.odps.create_table(test_table, Schema.from_lists(['num'], ['bigint']))
self.odps.write_table(t, data)
instance = self.odps.run_sql("select sum({0}(num)), 1 + '1' as warn_col from {1} group by num"
.format(function_name, test_table))
try:
self.assertEqual(instance.status, Instance.Status.RUNNING)
self.assertIn(instance.id, [it.id for it in self.odps.get_project().instances.iterate(
status=Instance.Status.RUNNING,
from_time=datetime.now()-timedelta(days=2),
end_time=datetime.now()+timedelta(days=1), max_items=20)])
self.waitContainerFilled(lambda: instance.tasks)
task = instance.tasks[0]
task.put_info('testInfo', 'TestInfo')
self.assertIsNotNone(task.warnings)
self.waitContainerFilled(lambda: task.workers, 30)
self.assertIsNotNone(task.workers[0].get_log('stdout'))
finally:
try:
instance.stop()
except:
pass
res.drop()
fun.drop()
t.drop()
def testRawUploadDownloadGreenlet(self):
block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))
partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME)
with partition.open_writer() as writer:
writer.write(TEST_FILE_NAME, block)
with partition.open_reader(TEST_FILE_NAME) as reader:
assert reader.read() == block
def testRawUploadDownloadGreenlet(self):
block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))
partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME)
with partition.open_writer() as writer:
writer.write(TEST_FILE_NAME, block)
with partition.open_reader(TEST_FILE_NAME) as reader:
assert reader.read() == block
def testZLibUploadDownload(self):
block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))
comp_option = CompressOption(level=9)
partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME)
with partition.open_writer(compress_option=comp_option) as writer:
writer.write(TEST_FILE_NAME, block, compress=True)
with partition.open_reader(TEST_FILE_NAME, compress_option=comp_option) as reader:
assert reader.read() == block
def testZLibUploadDownload(self):
block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))
comp_option = CompressOption(level=9)
partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME)
with partition.open_writer(compress_option=comp_option) as writer:
writer.write(TEST_FILE_NAME, block, compress=True)
with partition.open_reader(TEST_FILE_NAME, compress_option=comp_option) as reader:
assert reader.read() == block
def testRawUploadDownloadThread(self):
from odps.tunnel import io
io._FORCE_THREAD = True
block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))
partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME)
with partition.open_writer() as writer:
writer.write(TEST_FILE_NAME, block)
with partition.open_reader(TEST_FILE_NAME) as reader:
assert reader.read() == block
def testRawUploadDownloadThread(self):
from odps.tunnel import io
io._FORCE_THREAD = True
block = bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))
partition = self.odps.get_volume_partition(TEST_VOLUME_NAME, TEST_PARTITION_NAME)
with partition.open_writer() as writer:
writer.write(TEST_FILE_NAME, block)
with partition.open_reader(TEST_FILE_NAME) as reader:
assert reader.read() == block
def testListInstancesInPage(self):
test_table = tn('pyodps_t_tmp_list_instances_in_page')
data = [[random.randint(0, 1000)] for _ in compat.irange(10000)]
self.odps.delete_table(test_table, if_exists=True)
t = self.odps.create_table(test_table, Schema.from_lists(['num'], ['bigint']))
self.odps.write_table(t, data)
instance = self.odps.run_sql('select sum(num) from {0} group by num'.format(test_table))
try:
self.assertEqual(instance.status, Instance.Status.RUNNING)
self.assertIn(instance.id, [it.id for it in self.odps.get_project().instances.iterate(
status=Instance.Status.RUNNING,
from_time=datetime.now()-timedelta(days=2),
end_time=datetime.now()+timedelta(days=1), max_items=20)])
finally:
try:
instance.stop()
except:
pass
t.drop()
def testListInstancesInPage(self):
test_table = tn('pyodps_t_tmp_list_instances_in_page')
delay_udf = textwrap.dedent("""
from odps.udf import annotate
import sys
import time
@annotate("bigint->bigint")
class Delayer(object):
def evaluate(self, arg0):
print('Start Logging')
sys.stdout.flush()
time.sleep(45)
print('End Logging')
sys.stdout.flush()
return arg0
""")
resource_name = tn('test_delayer_function_resource')
function_name = tn('test_delayer_function')
if self.odps.exist_resource(resource_name + '.py'):
self.odps.delete_resource(resource_name + '.py')
res = self.odps.create_resource(resource_name + '.py',
'py',
file_obj=delay_udf)
if self.odps.exist_function(function_name):
self.odps.delete_function(function_name)
fun = self.odps.create_function(function_name,
class_type=resource_name + '.Delayer',
resources=[
res,
])
data = [[random.randint(0, 1000)] for _ in compat.irange(100)]
self.odps.delete_table(test_table, if_exists=True)
t = self.odps.create_table(test_table,
Schema.from_lists(['num'], ['bigint']))
self.odps.write_table(t, data)
instance = self.odps.run_sql(
"select sum({0}(num)), 1 + '1' as warn_col from {1} group by num".
format(function_name, test_table))
try:
self.assertEqual(instance.status, Instance.Status.RUNNING)
self.assertIn(instance.id, [
it.id for it in self.odps.get_project().instances.iterate(
status=Instance.Status.RUNNING,
start_time=datetime.now() - timedelta(days=2),
end_time=datetime.now() + timedelta(days=1),
max_items=20)
])
self.waitContainerFilled(lambda: instance.tasks)
task = instance.tasks[0]
task.put_info('testInfo', 'TestInfo')
self.assertIsNotNone(task.warnings)
self.waitContainerFilled(lambda: task.workers, 30)
self.assertIsNotNone(task.workers[0].get_log('stdout'))
finally:
try:
instance.stop()
except:
pass
res.drop()
fun.drop()
t.drop()
def _gen_byte_block():
return bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))
def _gen_byte_block():
return bytes(bytearray([iid % TEST_MODULUS for iid in irange(TEST_BLOCK_SIZE)]))